<commit_before>import numpy as np
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import matthews_corrcoef, mean_squared_error

from metal.metrics import metric_score


def mse(gold, _, probs):
    return {"mse": float(mean_squared_error(gold, probs))}


def spearman_corr(gold, _, probs):
    # NOTE: computes using "probs", which is the non-rounded output of the model.
    # TODO: fix this poor naming convention to better support regression tasks.
    probs_array = np.vstack(probs).squeeze()
    corr, p_value = spearmanr(gold.view(-1), probs_array)
    if np.isnan(corr):
        print(f"Warning: converting nan -> 0.0 for spearman_corr")
        corr = 0.0
    return {"spearman_corr": corr}


def pearson_corr(gold, _, probs):
    # NOTE: computes using "probs", which is the non-rounded output of the model.
    # TODO: fix this poor naming convention to better support regression tasks.
    probs_array = np.vstack(probs).squeeze()
    corr, p_value = pearsonr(gold.view(-1), probs_array)
    if np.isnan(corr):
        print(f"Warning: converting nan -> 0.0 for spearman_corr")
        corr = 0.0
    return {"pearson_corr": corr}


def matthews_corr(gold, outputs, **kwargs):
    return {"matthews_corr": matthews_corrcoef(gold, outputs)}


def acc_f1(gold, outputs, **kwargs):
    """A convenience custom function that returns accuracy, f1, and their mean"""
    accuracy = metric_score(gold, outputs, metric="accuracy")
    f1 = metric_score(gold, outputs, metric="f1")
    return {"accuracy": accuracy, "f1": f1, "acc_f1": np.mean([accuracy, f1])}


def ranking_acc_f1(gold, outputs, probs):
    """A convenience custom function that returns accuracy, f1, and their mean for ranking task heads."""
    gold = (1 - gold) + 1
    outputs = 1 * (probs.reshape((-1,)) > 0.5)
    accuracy = metric_score(gold, outputs, metric="accuracy")
    f1 = metric_score(gold, outputs, metric="f1")<commit_msg>Fix the fix that fixed the fix for pearson/spearman<commit_after>import numpy as np
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import matthews_corrcoef, mean_squared_error

from metal.metrics import metric_score


def mse(gold, _, probs):
    return {"mse": float(mean_squared_error(gold, probs))}


def spearman_corr(gold, _, probs):
    # NOTE: computes using "probs", which is the non-rounded output of the model.
    # TODO: fix this poor naming convention to better support regression tasks.
    probs_array = np.vstack(probs).squeeze()
    gold_array = np.vstack(gold).squeeze()
    corr, p_value = spearmanr(gold_array, probs_array)
    if np.isnan(corr):
        print(f"Warning: converting nan -> 0.0 for spearman_corr")
        corr = 0.0
    return {"spearman_corr": corr}


def pearson_corr(gold, _, probs):
    # NOTE: computes using "probs", which is the non-rounded output of the model.
    # TODO: fix this poor naming convention to better support regression tasks.
    probs_array = np.vstack(probs).squeeze()
    gold_array = np.vstack(gold).squeeze()
    corr, p_value = pearsonr(gold_array, probs_array)
    if np.isnan(corr):
        print(f"Warning: converting nan -> 0.0 for spearman_corr")
        corr = 0.0
    return {"pearson_corr": corr}


def matthews_corr(gold, outputs, **kwargs):
    return {"matthews_corr": matthews_corrcoef(gold, outputs)}


def acc_f1(gold, outputs, **kwargs):
    """A convenience custom function that returns accuracy, f1, and their mean"""
    accuracy = metric_score(gold, outputs, metric="accuracy")
    f1 = metric_score(gold, outputs, metric="f1")
    return {"accuracy": accuracy, "f1": f1, "acc_f1": np.mean([accuracy, f1])}


def ranking_acc_f1(gold, outputs, probs):
    """A convenience custom function that returns accuracy, f1, and their mean for ranking task heads."""
    gold = (1 - gold) + 1
    outputs = 1 * (probs.reshape((-1,)) > 0.5)
    accuracy = metric_score(gold, outputs, metric="accuracy")
    f1 = metric_score(gold, outputs, metric="f1")
{'fraction_non_alphanumeric': 0.12065491183879093, 'fraction_numerical': 0.013602015113350126, 'mean_word_length': 4.052162849872774, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1851345', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1290, 'n_words': 389}