<commit_before># -*- coding: utf-8 -*-

from functools import reduce
from itertools import product
from copy import copy, deepcopy
from collections import Mapping
from .util import reorder, hashlist, deprecated

from .storage import PickleStorage

__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


class Hasher(object):
    """
    metrics: iterable of metric_name
    domain: mapping param_name -> iterable of values (axis domain)
    metadata: mapping param_name -> param_val
    """
    @classmethod
    def sort_back(cls, dictionary):
        tmp = [(v, k) for k, v in dictionary.items()]
        tmp.sort()
        return [x for _, x in tmp]

    def __init__(self, metrics, domain, metadata=None):
        self.metric_inv = {p: i for i, p in enumerate(metrics)}
        self.dom_inv = {}
        last_stride = 1
        self.strides = {}
        for name, vals in domain.items():
            if len(vals) == 0:
                raise ValueError("Domain of '{}' is empty. "
                                 "If it is not useful remove it.".format(name))
            inv = {p: i for i, p in enumerate(vals)}
            self.dom_inv[name] = inv
            self.strides[name] = last_stride
            last_stride *= len(vals)

        self.metric_stride = last_stride
        if metadata is not None:
            for name, val in metadata.items():
                self.strides[name] = 0
                self.dom_inv[name] = {val: 0}

    def get_cons_args(self):
        metrics = Hasher.sort_back(self.metric_inv)
        metadata = {}
        domain = {}
        for param, domdic in self.dom_inv.items():
            if self.strides[param] == 0:
                metadata[param] = Hasher.sort_back(domdic)[0]
            else:
                domain[param] = Hasher.sort_back(domdic)
        if len(metadata) == 0:
            metadata = None
        return metrics, domain, metadata

    def __repr__(self):
        metrics, domain, metadata = self.get_cons_args()
        cls_name = self.__class__.__name__
        return "%s(metrics=%s, domain=%s, metadata=%s)" % (cls_name,
                                                           repr(metrics),
                                                           repr(domain),
                                                           repr(metadata))

    def add_metadata(self, **kwargs):
        for k, v in kwargs.items():
            self.strides[k] = 0
            self.dom_inv[k] = {v:0}

    def hash(self, metric, params):
        """
        metric: str
            A metric name
        params: mapping str -> value
            A mapping from parameter names to values of their domain
        """
        if len(params) != len(self.strides):
            raise IndexError("Expecting %d parameters/metadata, got %d" % (len(self.strides), len(params)))
        index = self.metric_inv[metric] * self.metric_stride
        for p, pv in params.items():
            index += (self.strides[p] * self.dom_inv[p][pv])
        return index

    def dehash(self, index):
        res = {}
        # First, find the metric:
        metric_index = index // self.metric_stride
        metrics = Hasher.sort_back(self.metric_inv)
        res["metric"] = metrics[metric_index]

        # Then, find the parameters
        index = index % self.metric_stride
        dims = [(v, k) for k,v in self.strides.items()]
        dims.sort(reverse=True)
        for stride, param in dims:
            if stride == 0:
                # We arrived at the metadata
                continue
            dom = Hasher.sort_back(self.dom_inv[param])
            p_idx = index // stride
            res[param] = dom[p_idx]
            index = index % stride

        return res

    def __call__(self, metric, params):
        return self.hash(metric, params)


class Datacube(Mapping):
    """
    parameters_ls : iterable of mappings param_name -> value
        The parameters of the experiment. The indexing must be coherent with
        `results_ls`

    results_ls: iterable of mappings metric_name -> value
        The metric of the experiment. The indexing must be coherent with
        `parameters_ls`

    exp_name: str (default: '')
        The name of the experiment

    force: boolean (default: True)
        Whether to force the building of the cube. If False, will raise
        exception for ill-formed cube (e.g. a parameter with no value). If True,
        will try to enforce the building of the cube -- possibly by droping some
        parameters

    Instance variables
    ------------------
    name: str (default "")
        The experiment name
    metadata: mapping str -> value
        A dictionary containing, for each of the metadatum, the associated value
    domain: mapping str -> iterable of values
        A dictionary containing, for each domain name, the iterable of possible
        values
    parameters: iterable of str
        The name of each axis (with corresponding index)
    metrics: iterable of str
        The name of each metric. The metric is the last axis. Each index in that
        axis correspond to a metric in the order of `metrics`
    data: iterable
        The raw data
    hash: :class:`Hasher`
        The hasher to the data
    shape: tuple (of size `len(parameters)+1`)
        The shape of the result cube. The N-1 first axis are linked to the
        parameters and the last one to the metrics
    """
    def __init__(self, parameters_ls, results_ls, exp_name="", force=True,
                 autopacking=False):
        self.autopacking = autopacking
        param_tmp = {}
        # Build back the parameters domain
        if not force and len(parameters_ls) == 0:
            raise ValueError("Empty cube. Use 'force=True' to ignore this "
                             "issue.")
        for parameters in parameters_ls:
            if not force and len(parameters) == 0:
                raise ValueError("Parameter '{}' has an empty domain. "
                                 "Use 'force=True' to ignore this issue."
                                 "".format(name))
            for name, v in parameters.items():
                _set = param_tmp.get(name)
                if _set is None:
                    _set = set()
                    param_tmp[name] = _set
                _set.add(v)
        # Sort metadata from parameters
        metadata = {}
        parameter_list = []
        domain = {}
        for name, v in param_tmp.items():
            ls = [vi for vi in v]
            if len(ls) > 1:
                ls.sort()
                domain[name] = [str(x) for x in ls]
                parameter_list.append(name)
            elif len(ls) == 1:
                metadata[name] = str(ls[0])
            else:
                # len(ls) <= 0: This should be handled by the previous stage
                if not force:
                    raise ValueError("Parameter '{}' has an empty domain. If "
                                     "it is not useful, remove it".format(name))
                # else: drop that axis
        parameter_list.sort()

        # Build back the metric list
        _set = set()
        for res in results_ls:
            _set.update(res.keys())<commit_msg>FIX mismatch type of domain values when building the datacube<commit_after># -*- coding: utf-8 -*-

from functools import reduce
from itertools import product
from copy import copy, deepcopy
from collections import Mapping
from .util import reorder, hashlist, deprecated, sort_per_type

from .storage import PickleStorage

__author__ = "<NAME> <<EMAIL>>"
__copyright__ = "3-clause BSD License"


class Hasher(object):
    """
    metrics: iterable of metric_name
    domain: mapping param_name -> iterable of values (axis domain)
    metadata: mapping param_name -> param_val
    """
    @classmethod
    def sort_back(cls, dictionary):
        tmp = [(v, k) for k, v in dictionary.items()]
        tmp.sort()
        return [x for _, x in tmp]

    def __init__(self, metrics, domain, metadata=None):
        self.metric_inv = {p: i for i, p in enumerate(metrics)}
        self.dom_inv = {}
        last_stride = 1
        self.strides = {}
        for name, vals in domain.items():
            if len(vals) == 0:
                raise ValueError("Domain of '{}' is empty. "
                                 "If it is not useful remove it.".format(name))
            inv = {p: i for i, p in enumerate(vals)}
            self.dom_inv[name] = inv
            self.strides[name] = last_stride
            last_stride *= len(vals)

        self.metric_stride = last_stride
        if metadata is not None:
            for name, val in metadata.items():
                self.strides[name] = 0
                self.dom_inv[name] = {val: 0}

    def get_cons_args(self):
        metrics = Hasher.sort_back(self.metric_inv)
        metadata = {}
        domain = {}
        for param, domdic in self.dom_inv.items():
            if self.strides[param] == 0:
                metadata[param] = Hasher.sort_back(domdic)[0]
            else:
                domain[param] = Hasher.sort_back(domdic)
        if len(metadata) == 0:
            metadata = None
        return metrics, domain, metadata

    def __repr__(self):
        metrics, domain, metadata = self.get_cons_args()
        cls_name = self.__class__.__name__
        return "%s(metrics=%s, domain=%s, metadata=%s)" % (cls_name,
                                                           repr(metrics),
                                                           repr(domain),
                                                           repr(metadata))

    def add_metadata(self, **kwargs):
        for k, v in kwargs.items():
            self.strides[k] = 0
            self.dom_inv[k] = {v:0}

    def hash(self, metric, params):
        """
        metric: str
            A metric name
        params: mapping str -> value
            A mapping from parameter names to values of their domain
        """
        if len(params) != len(self.strides):
            raise IndexError("Expecting %d parameters/metadata, got %d" % (len(self.strides), len(params)))
        index = self.metric_inv[metric] * self.metric_stride
        for p, pv in params.items():
            index += (self.strides[p] * self.dom_inv[p][pv])
        return index

    def dehash(self, index):
        res = {}
        # First, find the metric:
        metric_index = index // self.metric_stride
        metrics = Hasher.sort_back(self.metric_inv)
        res["metric"] = metrics[metric_index]

        # Then, find the parameters
        index = index % self.metric_stride
        dims = [(v, k) for k,v in self.strides.items()]
        dims.sort(reverse=True)
        for stride, param in dims:
            if stride == 0:
                # We arrived at the metadata
                continue
            dom = Hasher.sort_back(self.dom_inv[param])
            p_idx = index // stride
            res[param] = dom[p_idx]
            index = index % stride

        return res

    def __call__(self, metric, params):
        return self.hash(metric, params)


class Datacube(Mapping):
    """
    parameters_ls : iterable of mappings param_name -> value
        The parameters of the experiment. The indexing must be coherent with
        `results_ls`

    results_ls: iterable of mappings metric_name -> value
        The metric of the experiment. The indexing must be coherent with
        `parameters_ls`

    exp_name: str (default: '')
        The name of the experiment

    force: boolean (default: True)
        Whether to force the building of the cube. If False, will raise
        exception for ill-formed cube (e.g. a parameter with no value). If True,
        will try to enforce the building of the cube -- possibly by droping some
        parameters

    Instance variables
    ------------------
    name: str (default "")
        The experiment name
    metadata: mapping str -> value
        A dictionary containing, for each of the metadatum, the associated value
    domain: mapping str -> iterable of values
        A dictionary containing, for each domain name, the iterable of possible
        values
    parameters: iterable of str
        The name of each axis (with corresponding index)
    metrics: iterable of str
        The name of each metric. The metric is the last axis. Each index in that
        axis correspond to a metric in the order of `metrics`
    data: iterable
        The raw data
    hash: :class:`Hasher`
        The hasher to the data
    shape: tuple (of size `len(parameters)+1`)
        The shape of the result cube. The N-1 first axis are linked to the
        parameters and the last one to the metrics
    """
    def __init__(self, parameters_ls, results_ls, exp_name="", force=True,
                 autopacking=False):
        self.autopacking = autopacking
        param_tmp = {}
        # Build back the parameters domain
        if not force and len(parameters_ls) == 0:
            raise ValueError("Empty cube. Use 'force=True' to ignore this "
                             "issue.")
        for parameters in parameters_ls:
            if not force and len(parameters) == 0:
                raise ValueError("Parameter '{}' has an empty domain. "
                                 "Use 'force=True' to ignore this issue."
                                 "".format(name))
            for name, v in parameters.items():
                _set = param_tmp.get(name)
                if _set is None:
                    _set = set()
                    param_tmp[name] = _set
                _set.add(v)
        # Sort metadata from parameters
        metadata = {}
        parameter_list = []
        domain = {}
        for name, v in param_tmp.items():
            ls = [vi for vi in v]
            if len(ls) > 1:
                ls = sort_per_type(ls)
                domain[name] = [str(x) for x in ls]
                parameter_list.append(name)
            elif len(ls) == 1:
                metadata[name] = str(ls[0])
            else:
                # len(ls) <= 0: This should be handled by the previous stage
                if not force:
                    raise ValueError("Parameter '{}' has an empty domain. If "
                                     "it is not useful, remove it".format(name))
                # else: drop that axis
        parameter_list.sort()

        # Build back the metric list
        _set = set()
        for res in results_ls:
            _set.update(res.keys())
{'fraction_non_alphanumeric': 0.09254410143329658, 'fraction_numerical': 0.0027563395810363835, 'mean_word_length': 1.7137247569184741, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708889', 'n_tokens_mistral': 3927, 'n_tokens_neox': 3791, 'n_words': 1338}