<commit_before>"""
Methods for scoring prediction results (CA, AUC, ...).

Examples
--------
>>> import Orange
>>> data = Orange.data.Table('iris')
>>> learner = Orange.classification.LogisticRegressionLearner()
>>> results = Orange.evaluation.TestOnTrainingData(data, [learner])

"""

import math
import sys

import numpy as np
import sklearn.metrics as skl_metrics
from Orange.misc.wrapper_meta import WrapperMeta

__all__ = ["CA", "Precision", "Recall", "F1", "PrecisionRecallFSupport", "AUC",
           "MSE", "RMSE", "MAE", "R2", "compute_CD", "graph_ranks", "LogLoss"]


class Score(metaclass=WrapperMeta):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.
    """
    separate_folds = False
    is_scalar = True

    def __new__(cls, results=None, **kwargs):
        self = super().__new__(cls)
        if results is not None:
            self.__init__()
            return self(results, **kwargs)
        else:
            return self

    def __call__(self, results, **kwargs):
        if self.separate_folds and results.score_by_folds and results.folds:
            scores = self.scores_by_folds(results, **kwargs)
            return self.average(scores)

        return self.compute_score(results, **kwargs)

    def average(self, scores):
        if self.is_scalar:
            return np.mean(scores, axis=0)
        return NotImplementedError

    def scores_by_folds(self, results, **kwargs):
        nfolds = len(results.folds)
        nmodels = len(results.predicted)
        if self.is_scalar:
            scores = np.empty((nfolds, nmodels), dtype=np.float64)
        else:
            scores = [None] * nfolds
        for fold in range(nfolds):
            fold_results = results.get_fold(fold)
            scores[fold] = self.compute_score(fold_results, **kwargs)
        return scores

    def compute_score(self, results):
        return NotImplementedError

    @staticmethod
    def from_predicted(results, score_function, **kwargs):
        return np.fromiter(
            (score_function(results.actual, predicted, **kwargs)
             for predicted in results.predicted),
            dtype=np.float64, count=len(results.predicted))


## Classification scores

class CA(Score):
    __wraps__ = skl_metrics.accuracy_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.accuracy_score)


class Precision(Score):
    __wraps__ = skl_metrics.precision_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.precision_score,
                                   average="weighted")


class Recall(Score):
    __wraps__ = skl_metrics.recall_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.recall_score,
                                   average="weighted")


class F1(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    target : int, optional (default=None)
        Value of class to report.

    Examples
    --------
    >>> Orange.evaluation.F1(results)
    array([ 0.9...])

    """
    __wraps__ = skl_metrics.f1_score

    def compute_score(self, results, target=None):
        if target is None:
            if len(results.domain.class_var.values) <= 2:
                return self.from_predicted(results, skl_metrics.f1_score, average='binary')
            else:
                return self.from_predicted(results, skl_metrics.f1_score, average='weighted')
        else:
            return np.fromiter(
                (skl_metrics.f1_score(results.actual, predicted, average=None)[target]
                 for predicted in results.predicted),
                dtype=np.float64, count=len(results.predicted))


class PrecisionRecallFSupport(Score):
    __wraps__ = skl_metrics.precision_recall_fscore_support
    is_scalar = False

    def compute_score(self, results):
        return self.from_predicted(
            results, skl_metrics.precision_recall_fscore_support)


class AUC(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    target : int, optional (default=None)
        Value of class to report.
    """
    __wraps__ = skl_metrics.roc_auc_score
    separate_folds = True

    def calculate_weights(self, results):
        classes = np.unique(results.actual)
        class_cases = [sum(results.actual == class_)
                       for class_ in classes]
        N = results.actual.shape[0]
        weights = np.array([c * (N - c) for c in class_cases])
        wsum = np.sum(weights)
        if wsum == 0:
            raise ValueError("Class variable has less than two values")
        else:
            return weights / wsum

    def multi_class_auc(self, results):
        classes = np.unique(results.actual)
        weights = self.calculate_weights(results)

        auc_array = np.array([np.fromiter(
            (skl_metrics.roc_auc_score(
                results.actual == class_,
                (predicted == class_).astype(float))
             for predicted in results.predicted),
            dtype=np.float64, count=len(results.predicted))
                              for class_ in classes])

        return np.sum(auc_array.T * weights, axis=1)

    def compute_score(self, results, target=None):
        domain = results.domain
        n_classes = len(domain.class_var.values)

        if n_classes < 2:
            raise ValueError("Class variable has less than two values")
        elif n_classes == 2:
            return self.from_predicted(results, skl_metrics.roc_auc_score)
        else:
            if target is None:
                return self.multi_class_auc(results)
            else:
                y = np.array(results.actual == target, dtype=int)
                return np.fromiter(
                    (skl_metrics.roc_auc_score(y, probabilities[:, target])
                     for probabilities in results.probabilities),
                    dtype=np.float64, count=len(results.predicted))


class LogLoss(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    eps : float
        Log loss is undefined for p=0 or p=1, so probabilities are
        clipped to max(eps, min(1 - eps, p)).

    normalize : bool, optional (default=True)
        If true, return the mean loss per sample.
        Otherwise, return the sum of the per-sample losses.
<commit_msg>In compute_score of class AUC in case of two classes, roc_auc_score got actual predictions instead of predicted probabilities, which resulted in wrong AUC.<commit_after>"""
Methods for scoring prediction results (CA, AUC, ...).

Examples
--------
>>> import Orange
>>> data = Orange.data.Table('iris')
>>> learner = Orange.classification.LogisticRegressionLearner()
>>> results = Orange.evaluation.TestOnTrainingData(data, [learner])

"""

import math
import sys

import numpy as np
import sklearn.metrics as skl_metrics
from Orange.misc.wrapper_meta import WrapperMeta

__all__ = ["CA", "Precision", "Recall", "F1", "PrecisionRecallFSupport", "AUC",
           "MSE", "RMSE", "MAE", "R2", "compute_CD", "graph_ranks", "LogLoss"]


class Score(metaclass=WrapperMeta):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.
    """
    separate_folds = False
    is_scalar = True

    def __new__(cls, results=None, **kwargs):
        self = super().__new__(cls)
        if results is not None:
            self.__init__()
            return self(results, **kwargs)
        else:
            return self

    def __call__(self, results, **kwargs):
        if self.separate_folds and results.score_by_folds and results.folds:
            scores = self.scores_by_folds(results, **kwargs)
            return self.average(scores)

        return self.compute_score(results, **kwargs)

    def average(self, scores):
        if self.is_scalar:
            return np.mean(scores, axis=0)
        return NotImplementedError

    def scores_by_folds(self, results, **kwargs):
        nfolds = len(results.folds)
        nmodels = len(results.predicted)
        if self.is_scalar:
            scores = np.empty((nfolds, nmodels), dtype=np.float64)
        else:
            scores = [None] * nfolds
        for fold in range(nfolds):
            fold_results = results.get_fold(fold)
            scores[fold] = self.compute_score(fold_results, **kwargs)
        return scores

    def compute_score(self, results):
        return NotImplementedError

    @staticmethod
    def from_predicted(results, score_function, **kwargs):
        return np.fromiter(
            (score_function(results.actual, predicted, **kwargs)
             for predicted in results.predicted),
            dtype=np.float64, count=len(results.predicted))


## Classification scores

class CA(Score):
    __wraps__ = skl_metrics.accuracy_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.accuracy_score)


class Precision(Score):
    __wraps__ = skl_metrics.precision_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.precision_score,
                                   average="weighted")


class Recall(Score):
    __wraps__ = skl_metrics.recall_score

    def compute_score(self, results):
        return self.from_predicted(results, skl_metrics.recall_score,
                                   average="weighted")


class F1(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    target : int, optional (default=None)
        Value of class to report.

    Examples
    --------
    >>> Orange.evaluation.F1(results)
    array([ 0.9...])

    """
    __wraps__ = skl_metrics.f1_score

    def compute_score(self, results, target=None):
        if target is None:
            if len(results.domain.class_var.values) <= 2:
                return self.from_predicted(results, skl_metrics.f1_score, average='binary')
            else:
                return self.from_predicted(results, skl_metrics.f1_score, average='weighted')
        else:
            return np.fromiter(
                (skl_metrics.f1_score(results.actual, predicted, average=None)[target]
                 for predicted in results.predicted),
                dtype=np.float64, count=len(results.predicted))


class PrecisionRecallFSupport(Score):
    __wraps__ = skl_metrics.precision_recall_fscore_support
    is_scalar = False

    def compute_score(self, results):
        return self.from_predicted(
            results, skl_metrics.precision_recall_fscore_support)


class AUC(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    target : int, optional (default=None)
        Value of class to report.
    """
    __wraps__ = skl_metrics.roc_auc_score
    separate_folds = True

    def calculate_weights(self, results):
        classes = np.unique(results.actual)
        class_cases = [sum(results.actual == class_)
                       for class_ in classes]
        N = results.actual.shape[0]
        weights = np.array([c * (N - c) for c in class_cases])
        wsum = np.sum(weights)
        if wsum == 0:
            raise ValueError("Class variable has less than two values")
        else:
            return weights / wsum

    def multi_class_auc(self, results):
        classes = np.unique(results.actual)
        weights = self.calculate_weights(results)

        auc_array = np.array([np.fromiter(
            (skl_metrics.roc_auc_score(
                results.actual == class_,
                (predicted == class_).astype(float))
             for predicted in results.predicted),
            dtype=np.float64, count=len(results.predicted))
                              for class_ in classes])

        return np.sum(auc_array.T * weights, axis=1)

    def compute_score(self, results, target=None):
        domain = results.domain
        n_classes = len(domain.class_var.values)

        if n_classes < 2:
            raise ValueError("Class variable has less than two values")
        elif n_classes == 2:
            return np.fromiter(
                    (skl_metrics.roc_auc_score(results.actual, probabilities[:, 1])
                     for probabilities in results.probabilities),
                    dtype=np.float64, count=len(results.predicted))
        else:
            if target is None:
                return self.multi_class_auc(results)
            else:
                y = np.array(results.actual == target, dtype=int)
                return np.fromiter(
                    (skl_metrics.roc_auc_score(y, probabilities[:, target])
                     for probabilities in results.probabilities),
                    dtype=np.float64, count=len(results.predicted))


class LogLoss(Score):
    """
    ${sklpar}
    Parameters
    ----------
    results : Orange.evaluation.Results
        Stored predictions and actual data in model testing.

    eps : float
        Log loss is undefined for p=0 or p=1, so probabilities are
        clipped to max(eps, min(1 - eps, p)).

    normalize : bool, optional (default=True)
        If true, return the mean loss per sample.
        Otherwise, return the sum of the per-sample losses.

{'fraction_non_alphanumeric': 0.10298713569300094, 'fraction_numerical': 0.004578821135256923, 'mean_word_length': 2.302135829133669, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1435059', 'n_tokens_mistral': 4007, 'n_tokens_neox': 3846, 'n_words': 960}