<commit_before>from collections import defaultdict

import numpy as np

from metal.logging.utils import join_full_metric, split_full_metric
from metal.metrics import METRICS as STANDARD_METRICS, metric_score
from metal.mmtl.utils.utils import stack_batches


"""
Scorer class which evaluates metrics given a task and model.

#TODO: Update this after redesign
Example usage 1: standalone
scorer = Scorer(standard_metrics=["accuracy"])
model = MetalModel(tasks)
trainer = Trainer()
trainer.train_model(model, tasks)
scorer.score(model, tasks[0])

Example usage 2: integrated into task
scorer = Scorer(standard_metrics=["accuracy"])
task1 = Task(...,scorer=scorer)
...
taskn = ...
tasks = [task1, ..., taskn]
model = MetalModel(tasks)
"""


class Scorer(object):
    """
    # TODO: clarify this even further
    DESIGN:
    - A Scorer is a bundle of metrics; it defines what metrics _can_ be calculated on a
    given task (may be able to use smart defaults based on the Task subclass; e.g.,
    classification comes with many nicely defined).
        - custom functions come with a list of names of the metrics they produce (with
        error checking to confirm they don't produce more than that)
    - A Scorer is applied to a task with a list of metrics to return
    - All metrics in a scorer produce simple metric name only
        - score() has task and split so it can create the full metric name
        - The metrics dict only ever contains full metric names
    - The user submits a list of task_metrics and test_metrics to the trainer
        - test_metrics defaults to task_metrics
        - when score() is called, only task_metrics are calculated and returned
        - task_metrics and test_metrics contain only full metric names
            - these can use arbitrary split names
        - [later] we can optionally allow regexes instead of explicit names

    Args:
        data_loader: DataLoader on which to calculate metrics.
        standard_metrics: List of strings of standard metrics for which to evaluate.
            By default, calculate on valid split. Optionally, prepend metric with
            "train/" to calculate on train split instead.
        custom_metric_funcs: Dict of the form:

            {metric_fn1: ["metric1a", ..., "metric1z"],
             metric_fn2: ["metric2a", ..., "metric2z]}

            where metric_fn is a function of the form:

            metric_fn1(Y, Y_preds, probs=Y_probs) ->
                {metric1a: value1, ..., metric1z: valueN}

            Note that metric_names will automatically have task and split prefixes
            added by the Scorer;
    """

    def __init__(self, standard_metrics=["accuracy"], custom_metric_funcs={}):
        self.standard_metrics = standard_metrics
        for metric_name in standard_metrics:
            if "/" in metric_name:
                msg = (
                    f"Standard metrics at Scorer initialization time must not "
                    "include task or split name, but you submitted: {metric_name}"
                )
                raise Exception(msg)
            if metric_name not in STANDARD_METRICS:
                msg = (
                    f"Requested standard metric {metric_name} could not be found in "
                    "metrics.py."<commit_msg>Remove "accuracy" from standard_metrics in Scorer; handle in Task subclasses<commit_after>from collections import defaultdict

import numpy as np

from metal.logging.utils import join_full_metric, split_full_metric
from metal.metrics import METRICS as STANDARD_METRICS, metric_score
from metal.mmtl.utils.utils import stack_batches


"""
Scorer class which evaluates metrics given a task and model.

#TODO: Update this after redesign
Example usage 1: standalone
scorer = Scorer(standard_metrics=["accuracy"])
model = MetalModel(tasks)
trainer = Trainer()
trainer.train_model(model, tasks)
scorer.score(model, tasks[0])

Example usage 2: integrated into task
scorer = Scorer(standard_metrics=["accuracy"])
task1 = Task(...,scorer=scorer)
...
taskn = ...
tasks = [task1, ..., taskn]
model = MetalModel(tasks)
"""


class Scorer(object):
    """
    # TODO: clarify this even further
    DESIGN:
    - A Scorer is a bundle of metrics; it defines what metrics _can_ be calculated on a
    given task (may be able to use smart defaults based on the Task subclass; e.g.,
    classification comes with many nicely defined).
        - custom functions come with a list of names of the metrics they produce (with
        error checking to confirm they don't produce more than that)
    - A Scorer is applied to a task with a list of metrics to return
    - All metrics in a scorer produce simple metric name only
        - score() has task and split so it can create the full metric name
        - The metrics dict only ever contains full metric names
    - The user submits a list of task_metrics and test_metrics to the trainer
        - test_metrics defaults to task_metrics
        - when score() is called, only task_metrics are calculated and returned
        - task_metrics and test_metrics contain only full metric names
            - these can use arbitrary split names
        - [later] we can optionally allow regexes instead of explicit names

    Args:
        data_loader: DataLoader on which to calculate metrics.
        standard_metrics: List of strings of standard metrics for which to evaluate.
            By default, calculate on valid split. Optionally, prepend metric with
            "train/" to calculate on train split instead.
        custom_metric_funcs: Dict of the form:

            {metric_fn1: ["metric1a", ..., "metric1z"],
             metric_fn2: ["metric2a", ..., "metric2z]}

            where metric_fn is a function of the form:

            metric_fn1(Y, Y_preds, probs=Y_probs) ->
                {metric1a: value1, ..., metric1z: valueN}

            Note that metric_names will automatically have task and split prefixes
            added by the Scorer;
    """

    def __init__(self, standard_metrics=[], custom_metric_funcs={}):
        self.standard_metrics = standard_metrics
        for metric_name in standard_metrics:
            if "/" in metric_name:
                msg = (
                    f"Standard metrics at Scorer initialization time must not "
                    "include task or split name, but you submitted: {metric_name}"
                )
                raise Exception(msg)
            if metric_name not in STANDARD_METRICS:
                msg = (
                    f"Requested standard metric {metric_name} could not be found in "
                    "metrics.py."
{'fraction_non_alphanumeric': 0.06789656736730681, 'fraction_numerical': 0.004536518977771057, 'mean_word_length': 2.60239651416122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512822', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1680, 'n_words': 726}