<commit_before>"""@author: <NAME> <<EMAIL>>"""

import numpy
import cargo

logger = cargo.get_logger(__name__, default_level = "INFO")

class RunRecord(object):
    """Record of a solver run."""

    def __init__(self, solver, budget, cost, success):
        """Initialize."""

        self.solver = solver
        self.budget = budget
        self.cost = cost
        self.success = success

class TrainingData(object):
    """Load and access portfolio training data."""

    def __init__(self, task_paths, domain, suffix):
        """Collect training data from task paths."""

        self._run_lists = {}
        self._feature_vectors = {}

        for path in task_paths:
            # load run records
            run_data = numpy.recfromcsv(path + suffix, usemask = True)
            run_list = []

            for (run_solver, run_budget, run_cost, run_succeeded, run_answer) in run_data.tolist():
                record = RunRecord(run_solver, run_budget, run_cost, run_succeeded)

                run_list.append(record)

            self._run_lists[path] = run_list

            # load feature data
            feature_vector = numpy.recfromcsv("{0}.features.csv".format(path)).tolist()

            self._feature_vectors[path] = feature_vector

    def get_run_list(self, id_):
        """Retrieve runs made on a task."""

        return self._run_lists[id_]

    def get_run_lists(self):
        """Retrieve runs made on all tasks."""

        return self._run_lists

    def get_feature_vector(self, id_):
        """Retrieve features of a task."""

        return self._feature_vectors[id_]

    def get_feature_vectors(self):
        """Retrieve features of all tasks."""

        return self._feature_vectors

    @staticmethod
    def from_roots(tasks_roots, domain, suffix = ".runs.csv"):
        """Collect training data by scanning for tasks."""

        task_paths = []

        for tasks_root in tasks_roots:
            task_paths.extend(cargo.files_under(tasks_root, domain.extensions))

        return TrainingData(task_paths)

def outcome_matrices_from_runs(solver_index, budgets, run_lists):
    """Build run-outcome matrices from records."""

    S = len(solver_index)
    B = len(budgets)
    N = len(run_lists)
    successes = numpy.zeros((N, S, B))
    attempts = numpy.zeros((N, S))

    for (n, (_, runs)) in enumerate(run_lists.items()):
        for run in runs:
            s = solver_index.get(run.solver)

            if s is not None and run.budget >= budgets[-1]:
                b = numpy.digitize([run.cost], budgets)

                attempts[n, s] += 1.0

                if b < B and run.success:
                    successes[n, s, b] += 1.0

    return (successes, attempts)

<commit_msg>Compensate for numpy behavior on single-row CSVs.<commit_after>"""@author: <NAME> <<EMAIL>>"""

import numpy
import cargo

logger = cargo.get_logger(__name__, default_level = "INFO")

class RunRecord(object):
    """Record of a solver run."""

    def __init__(self, solver, budget, cost, success):
        """Initialize."""

        self.solver = solver
        self.budget = budget
        self.cost = cost
        self.success = success

class TrainingData(object):
    """Load and access portfolio training data."""

    def __init__(self, task_paths, domain, suffix):
        """Collect training data from task paths."""

        self._run_lists = {}
        self._feature_vectors = {}

        for path in task_paths:
            # load run records
            run_list = []
            run_data = numpy.recfromcsv(path + suffix, usemask = True)
            rows = run_data.tolist()

            if run_data.shape == ():
                rows = [rows]

            for (run_solver, run_budget, run_cost, run_succeeded, run_answer) in rows:
                record = RunRecord(run_solver, run_budget, run_cost, run_succeeded)

                run_list.append(record)

            self._run_lists[path] = run_list

            # load feature data
            feature_vector = numpy.recfromcsv("{0}.features.csv".format(path)).tolist()

            self._feature_vectors[path] = feature_vector

    def get_run_list(self, id_):
        """Retrieve runs made on a task."""

        return self._run_lists[id_]

    def get_run_lists(self):
        """Retrieve runs made on all tasks."""

        return self._run_lists

    def get_feature_vector(self, id_):
        """Retrieve features of a task."""

        return self._feature_vectors[id_]

    def get_feature_vectors(self):
        """Retrieve features of all tasks."""

        return self._feature_vectors

    @staticmethod
    def from_roots(tasks_roots, domain, suffix = ".runs.csv"):
        """Collect training data by scanning for tasks."""

        task_paths = []

        for tasks_root in tasks_roots:
            task_paths.extend(cargo.files_under(tasks_root, domain.extensions))

        return TrainingData(task_paths)

def outcome_matrices_from_runs(solver_index, budgets, run_lists):
    """Build run-outcome matrices from records."""

    S = len(solver_index)
    B = len(budgets)
    N = len(run_lists)
    successes = numpy.zeros((N, S, B))
    attempts = numpy.zeros((N, S))

    for (n, (_, runs)) in enumerate(run_lists.items()):
        for run in runs:
            s = solver_index.get(run.solver)

            if s is not None and run.budget >= budgets[-1]:
                b = numpy.digitize([run.cost], budgets)

                attempts[n, s] += 1.0

                if b < B and run.success:
                    successes[n, s, b] += 1.0

    return (successes, attempts)


{'fraction_non_alphanumeric': 0.11472480343102216, 'fraction_numerical': 0.002144388849177984, 'mean_word_length': 2.329565734681737, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3713996', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1620, 'n_words': 434}