<commit_before># -*- coding: utf-8 -*-

"""
Module :mod:`experiment` is a set of functions and classes to build, run and
monitor experiments

Restriction
-----------
Experiment names should always elligible file names.
"""

from itertools import product
from copy import copy, deepcopy
from collections import Mapping, defaultdict

from functools import reduce
from six import string_types
from clusterlib.scheduler import submit

from .database import get_storage, load_results
from .notification import (pending_job_update, running_job_update,
                           completed_job_update, aborted_job_update,
                           partial_job_update, critical_job_update, Historic)
from .util import reorder, hashlist


__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


__EXP_NAME__ = "Experiment"
__PARAMETERS__ = "Parameters"
__RESULTS__ = "Results"


class Computation(object):

    def __init__(self, exp_name, comp_name, overwrite=True):
        self.exp_name = exp_name
        self.comp_name = comp_name
        self.overwrite = overwrite
        self.storage = get_storage(self.exp_name)

    def _save(self, result):
        self.storage.save_result(self.comp_name, result, self.overwrite)

    def run(self, **parameters):
        pass

    def __call__(self, **parameters):
        start = running_job_update(self.exp_name, self.comp_name)
        try:
            result = {
                self.comp_name: {
                    __EXP_NAME__: self.exp_name,
                    __PARAMETERS__: parameters,
                    __RESULTS__: self.run(**parameters)
                }
            }
            critical_job_update(self.exp_name, self.comp_name, start)
            self._save(result)
            completed_job_update(self.exp_name, self.comp_name, start)
        except Exception as excep:
            aborted_job_update(self.exp_name, self.comp_name, start, excep)
            raise

class PartialComputation(Computation):
    """
    Expect the run method to be a generator
    """
    def __call__(self, **parameters):
        start = running_job_update(self.exp_name, self.comp_name)
        try:
            for partial_result in self.run(**parameters):
                result = {
                    self.comp_name: {
                        __EXP_NAME__: self.exp_name,
                        __PARAMETERS__: parameters,
                        __RESULTS__: partial_result
                    }
                }
                critical_job_update(self.exp_name, self.comp_name, start)
                self._save(result)
                partial_job_update(self.exp_name, self.comp_name, start)
            completed_job_update(self.exp_name, self.comp_name, start)
        except Exception as excep:
            aborted_job_update(self.exp_name, self.comp_name, start, excep)
            raise

    def load_partial(self):
        """return a dict"""
        return self.storage.load_result(self.comp_name)


class Experiment(object):
    """
    param_seq : list of dict
    """
    def __init__(self, name, params=None, param_seq=None):
        self.name = name
        if not params:
            self.params = {}
        else:
            self.params = params
        if param_seq:
            self.param_seq = [copy(self.params)] + list(param_seq)
            self.params = self.param_seq[-1]
        else:
            self.param_seq = [copy(self.params)]

    def add_params(self, **kwargs):
        for k, v in kwargs.items():
            vp = self.params.get(k)
            if vp is None:
                if len(self.param_seq) > 1:
                    raise ValueError("New keyword (i.e. '%s') cannot be addded "
                                     "after the first barrier" % str(k))
                vp = set()
                self.params[k] = vp
            vps = self.param_seq[-1].get(k)
            if vps is None:
                vps = set()
                self.param_seq[-1][k] = vps
            try:
                if isinstance(v, string_types):
                    raise TypeError
                for vi in v:
                    if not vi in vp:
                        vp.add(vi)
                        vps.add(vi)
            except TypeError:
                if v not in vp:
                    vp.add(v)
                    vps.add(v)
        return self

    def add_separator(self):
        self.param_seq.append({k:set() for k in self.param_seq[0].keys()})
        return self

    def __len__(self):
        nb = 1
        for val in self.params.values():
            nb *= len(val)
        return nb

    def __iter__(self):
        # TODO less memory by factorization
        if len(self.params) == 0:
            yield "Computation-"+self.name+"-0", {}
        else:
            keys = self.params.keys()
            keys.sort()
            inc = {k:set() for k in keys}<commit_msg>COMPAT dict.keys is not a list in python3 so makes it a list.<commit_after># -*- coding: utf-8 -*-

"""
Module :mod:`experiment` is a set of functions and classes to build, run and
monitor experiments

Restriction
-----------
Experiment names should always elligible file names.
"""

from itertools import product
from copy import copy, deepcopy
from collections import Mapping, defaultdict

from functools import reduce
from six import string_types
from clusterlib.scheduler import submit

from .database import get_storage, load_results
from .notification import (pending_job_update, running_job_update,
                           completed_job_update, aborted_job_update,
                           partial_job_update, critical_job_update, Historic)
from .util import reorder, hashlist


__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


__EXP_NAME__ = "Experiment"
__PARAMETERS__ = "Parameters"
__RESULTS__ = "Results"


class Computation(object):

    def __init__(self, exp_name, comp_name, overwrite=True):
        self.exp_name = exp_name
        self.comp_name = comp_name
        self.overwrite = overwrite
        self.storage = get_storage(self.exp_name)

    def _save(self, result):
        self.storage.save_result(self.comp_name, result, self.overwrite)

    def run(self, **parameters):
        pass

    def __call__(self, **parameters):
        start = running_job_update(self.exp_name, self.comp_name)
        try:
            result = {
                self.comp_name: {
                    __EXP_NAME__: self.exp_name,
                    __PARAMETERS__: parameters,
                    __RESULTS__: self.run(**parameters)
                }
            }
            critical_job_update(self.exp_name, self.comp_name, start)
            self._save(result)
            completed_job_update(self.exp_name, self.comp_name, start)
        except Exception as excep:
            aborted_job_update(self.exp_name, self.comp_name, start, excep)
            raise

class PartialComputation(Computation):
    """
    Expect the run method to be a generator
    """
    def __call__(self, **parameters):
        start = running_job_update(self.exp_name, self.comp_name)
        try:
            for partial_result in self.run(**parameters):
                result = {
                    self.comp_name: {
                        __EXP_NAME__: self.exp_name,
                        __PARAMETERS__: parameters,
                        __RESULTS__: partial_result
                    }
                }
                critical_job_update(self.exp_name, self.comp_name, start)
                self._save(result)
                partial_job_update(self.exp_name, self.comp_name, start)
            completed_job_update(self.exp_name, self.comp_name, start)
        except Exception as excep:
            aborted_job_update(self.exp_name, self.comp_name, start, excep)
            raise

    def load_partial(self):
        """return a dict"""
        return self.storage.load_result(self.comp_name)


class Experiment(object):
    """
    param_seq : list of dict
    """
    def __init__(self, name, params=None, param_seq=None):
        self.name = name
        if not params:
            self.params = {}
        else:
            self.params = params
        if param_seq:
            self.param_seq = [copy(self.params)] + list(param_seq)
            self.params = self.param_seq[-1]
        else:
            self.param_seq = [copy(self.params)]

    def add_params(self, **kwargs):
        for k, v in kwargs.items():
            vp = self.params.get(k)
            if vp is None:
                if len(self.param_seq) > 1:
                    raise ValueError("New keyword (i.e. '%s') cannot be addded "
                                     "after the first barrier" % str(k))
                vp = set()
                self.params[k] = vp
            vps = self.param_seq[-1].get(k)
            if vps is None:
                vps = set()
                self.param_seq[-1][k] = vps
            try:
                if isinstance(v, string_types):
                    raise TypeError
                for vi in v:
                    if not vi in vp:
                        vp.add(vi)
                        vps.add(vi)
            except TypeError:
                if v not in vp:
                    vp.add(v)
                    vps.add(v)
        return self

    def add_separator(self):
        self.param_seq.append({k:set() for k in self.param_seq[0].keys()})
        return self

    def __len__(self):
        nb = 1
        for val in self.params.values():
            nb *= len(val)
        return nb

    def __iter__(self):
        # TODO less memory by factorization
        if len(self.params) == 0:
            yield "Computation-"+self.name+"-0", {}
        else:
            keys = list(self.params.keys())
            keys.sort()
            inc = {k:set() for k in keys}
{'fraction_non_alphanumeric': 0.09329802403748218, 'fraction_numerical': 0.0021389284986759014, 'mean_word_length': 1.7282578494026117, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '892740', 'n_tokens_mistral': 2843, 'n_tokens_neox': 2725, 'n_words': 695}