<commit_before>#
# Lancet core
#

import os, time, itertools, copy
import re, glob, string, re
import json

import param

try:
    import numpy as np
    np_ftypes = np.sctypes['float']
except:
    np, np_ftypes = None, []

try:    from pandas import DataFrame
except: DataFrame = None

from collections import defaultdict

float_types = [float] + np_ftypes
def identityfn(x): return x
def fp_repr(x):    return str(x) if (type(x) in float_types) else repr(x)

def set_fp_precision(value):
    """
    Function to set the floating precision across lancet.
    """
    BaseArgs.set_default('fp_precision', value)

#=====================#
# Argument Specifiers #
#=====================#

class BaseArgs(param.Parameterized):
    """
    The base class for all argument specifiers. Argument specifiers implement
    Python's iterator protocol, returning arguments on each call to next(). Note
    that these objects should not be written as generators as they often need to
    be pickled.
    """

    fp_precision = param.Integer(default=4, constant=True, doc='''
         The floating point precision to use for floating point values.  Unlike
         other basic Python types, floats need care with their representation as
         you only want to display up to the precision actually specified.''')

    def __init__(self, **params):
        self._pprint_args = ([],[],None,{})
        super(BaseArgs,self).__init__(**params)
        self.pprint_args([],['fp_precision', 'dynamic'])

    def __iter__(self): return self

    def __contains__(self, value):
        return value in (self.constant_keys + self.varying_keys)

    @classmethod
    def spec_formatter(cls, spec):
        " Formats the elements of an argument set appropriately"
        return dict((k, str(v)) for (k,v) in spec.items())

    @property
    def constant_keys(self):
        """
        Returns the list of parameter names whose values are constant as the
        argument specifier is iterated.  Note that the union of constant and
        varying_keys should partition the entire set of keys.
        """
        raise NotImplementedError

    @property
    def constant_items(self):
        """
        Returns the set of constant items as a list of tuples. This allows easy
        conversion to dictionary format. Note, the items should be supplied in
        the same key ordering as for constant_keys for consistency.
        """
        raise NotImplementedError

    @property
    def varying_keys(self):
        """
        Returns the list of parameters whose values vary as the argument
        specifier is iterated.  Whenever it is possible, keys should be sorted
        from those slowest to faster varying and sorted alphanumerically within
        groups that vary at the same rate.
        """
        raise NotImplementedError

    def round_floats(self, specs, fp_precision):
        _round_float = lambda v, fp: np.round(v, fp) if (type(v) in np_ftypes) else round(v, fp)
        _round = (lambda v, fp: int(v)) if fp_precision==0  else _round_float
        return (dict((k, _round(v, fp_precision) if (type(v) in float_types) else v)
                     for (k,v) in spec.items()) for spec in specs)

    def next(self):
        """
        Called to get a list of specifications: dictionaries with parameter name
        keys and string values.
        """
        raise StopIteration

    def copy(self):
        """
        Convenience method to avoid using the specifier without exhausting it.
        """
        return copy.copy(self)

    def _collect_by_key(self,specs):
        """
        Returns a dictionary like object with the lists of values collapsed by
        their respective key. Useful to find varying vs constant keys and to
        find how fast keys vary.
        """
        # Collects key, value tuples as list of lists then flatten using chain
        allkeys = itertools.chain.from_iterable([[(k, run[k])  for k in run] for run in specs])
        collection = defaultdict(list)
        for (k,v) in allkeys: collection[k].append(v)
        return collection

    def __add__(self, other):
        """
        Concatenates two argument specifiers.
        """
        if not other: return self
        return Concatenate(self,other)

    def __mul__(self, other):
        """
        Takes the Cartesian product of two argument specifiers.
        """
        return CartesianProduct(self, other)

    def __radd__(self, other):
        if not other: return self

    def __rmul__(self, other):
        if not other: return []

    def _cartesian_product(self, first_specs, second_specs):
        """
        Takes the Cartesian product of the specifications. Result will contain N
        specifications where N = len(first_specs) * len(second_specs) and keys
        are merged.  Example: [{'a':1},{'b':2}] * [{'c':3},{'d':4}] =
        [{'a':1,'c':3},{'a':1,'d':4},{'b':2,'c':3},{'b':2,'d':4}]
        """
        return  [ dict(zip(
                          list(s1.keys()) + list(s2.keys()),
                          list(s1.values()) + list(s2.values())
                      ))
                 for s1 in first_specs for s2 in second_specs ]

    def pprint_args(self, pos_args, keyword_args, infix_operator=None, extra_params={}):
        """
        Method to define the positional arguments and keyword order for pretty printing.
        """
        if infix_operator and not (len(pos_args)==2 and keyword_args==[]):
            raise Exception('Infix format requires exactly two positional arguments and no keywords')
        (kwargs,_,_,_) = self._pprint_args
        self._pprint_args = (keyword_args + kwargs, pos_args, infix_operator, extra_params)

    def _pprint(self, cycle=False, flat=False, annotate=False, level=1, tab = '   '):
        """
        Pretty printer that prints only the modified keywords and generates flat
        representations (for repr) and optionally annotates with a comment.
        """
        (kwargs, pos_args, infix_operator, extra_params) = self._pprint_args
        (br, indent)  = ('' if flat else '\n', '' if flat else tab * level)
        prettify = lambda x: isinstance(x, BaseArgs) and not flat
        pretty = lambda x: x._pprint(flat=flat, level=level+1) if prettify(x) else repr(x)

        params = dict(self.get_param_values())
        modified = [k for (k,v) in self.get_param_values(onlychanged=True)]
        pkwargs = [(k, params[k])  for k in kwargs if (k in modified)] + extra_params.items()
        arg_list = [(k,params[k]) for k in pos_args] + pkwargs

        len_ckeys, len_vkeys = len(self.constant_keys), len(self.varying_keys)
        info_triple = (len(self),
                       ', %d constant key(s)' % len_ckeys if len_ckeys else '',
                       ', %d varying key(s)'  % len_vkeys if len_vkeys else '')
        annotation = '# == %d items%s%s ==\n' % info_triple
        lines = [annotation] if annotate else []    # Optional annotating comment

        if cycle:
            lines.append('%s(...)' % self.__class__.__name__)
        elif infix_operator:
            level = level - 1
            triple = (pretty(params[pos_args[0]]), infix_operator, pretty(params[pos_args[1]]))
            lines.append('%s %s %s' % triple)
        else:
            lines.append('%s(' % self.__class__.__name__)
            for (k,v) in arg_list:
                lines.append('%s%s=%s' %  (br+indent, k, pretty(v)))
                lines.append(',')
            lines = lines[:-1] +[br+(tab*(level-1))+')'] # Remove trailing comma

        return ''.join(lines)

    def __repr__(self):  return self._pprint(flat=True)
    def __str__(self): return self._pprint()

class Args(BaseArgs):

    specs = param.List(default=[], constant=True, doc='''
          The static list of specifications (ie. dictionaries) to be
          returned by the specifier. Float values are rounded to
          fp_precision.''')

    def __init__(self, specs=None, fp_precision=None, **kwargs):
        if fp_precision is None: fp_precision = BaseArgs.fp_precision
        raw_specs, kwargs, explicit = self._build_specs(specs, kwargs, fp_precision)
        super(Args, self).__init__(fp_precision=fp_precision, specs=raw_specs, **kwargs)

        self._lexorder = None
        # Some types cannot be sorted (e.g. numpy arrays)
        self.unsortable_keys = []
        if explicit: 
            self.pprint_args(['specs'],[])
        else: # Present in kwarg format
            self.pprint_args([], self.constant_keys, None, 
                             dict(self.constant_items))

    def _build_specs(self, specs, kwargs, fp_precision):
        """
        Returns the specs, the remaining kwargs and whether or not the
        constructor was called with kwarg or explicit specs.
        """
        if isinstance(specs, list):
            return list(self.round_floats(specs, fp_precision)), kwargs, True
        elif specs is None:
            overrides = param.ParamOverrides(self, kwargs, allow_extra_keywords=True)
            extra_kwargs = overrides.extra_keywords()
            kwargs = dict([(k,v) for (k,v) in kwargs.items() if k not in extra_kwargs])
            return list(self.round_floats([extra_kwargs], fp_precision)), kwargs, False

    def __iter__(self):
        self._exhausted = False
        return self

    def next(self):
        if self._exhausted:
            raise StopIteration
        else:
            self._exhausted=True
            return self.specs

    def _unique(self, sequence, idfun=repr):
        """
        Note: repr() must be implemented properly on all objects. This is
        assumed by lancet when Python objects need to be formatted to string
        representation.
        """
        seen = {}
        return [seen.setdefault(idfun(e),e) for e in sequence
                if idfun(e) not in seen]

    def show(self):
        """
        Convenience method to inspect the available argument values in
        human-readable format. The ordering of keys is determined by
        how quickly they vary.
        """
        ordering = self.constant_keys + self.varying_keys
        spec_lines = [', '.join(['%s=%s' % (k, s[k]) for k in ordering if k in s]) for s in self.specs]
        print('\n'.join(['%d: %s' % (i,l) for (i,l) in enumerate(spec_lines)]))

    @property
    def constant_keys(self):
        collection = self._collect_by_key(self.specs)
        return [k for k in sorted(collection) if (len(self._unique(collection[k])) == 1)]

    @property
    def constant_items(self):
        collection = self._collect_by_key(self.specs)
        return [(k,collection[k][0]) for k in self.constant_keys]

    @property
    def varying_keys(self):
        collection = self._collect_by_key(self.specs)
        constant_set = set(self.constant_keys)
        unordered_varying = set(collection.keys()).difference(constant_set)
        # Finding out how fast keys are varying
        grouplens = [(len([len(list(y)) for (_,y) in itertools.groupby(collection[k])]),k) for k in collection]
        varying_counts = [(n,k) for (n,k) in sorted(grouplens) if (k in unordered_varying)]
        # Grouping keys with common frequency alphanumerically (desired behaviour).
        ddict = defaultdict(list)
        for (n,k) in varying_counts: ddict[n].append(k)
        alphagroups = [sorted(ddict[k]) for k in sorted(ddict)]
        return [el for group in alphagroups for el in group]

    @property
    def dframe(self):
        return DataFrame(self.specs) if DataFrame else "Pandas not available"

    def __len__(self): return len(self.specs)


class Concatenate(Args):
    """
    Concatenate is the sequential composition of two specifiers. The
    specifier created by the compositon (firsts + second) generates
    the arguments in first followed by the arguments in second.
    """

    first = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The first specifier used to generate the concatenation.''')

    second = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The second specifier used to generate the concatenation.''')

    def __init__(self, first, second):

        max_precision = max(first.fp_precision, second.fp_precision)
        specs = first.specs + second.specs
        super(Concatenate, self).__init__(specs, fp_precision=max_precision,
                                                first=first, second=second)
        self.pprint_args(['first', 'second'],[], infix_operator='+')

class CartesianProduct(Args):
    """
    CartesianProduct is the cartesian product of two specifiers. The
    specifier created by the compositon (firsts * second) generates
    the cartesian produce of the arguments in first followed by the
    arguments in second. Note that len(first * second) =
    len(first)*len(second)
    """

    first = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The first specifier used to generate the Cartesian product.''')

    second = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The second specifier used to generate the Cartesian product.''')

    def __init__(self, first, second):

        max_precision = max(first.fp_precision, second.fp_precision)
        specs = self._cartesian_product(first.specs, second.specs)

        overlap = (set(first.varying_keys + first.constant_keys)
                   &  set(second.varying_keys + second.constant_keys))
        assert overlap == set(), 'Sets of keys cannot overlap between argument specifiers in cartesian product.'
        super(CartesianProduct, self).__init__(specs, fp_precision=max_precision,
                                               first=first, second=second)
        self.pprint_args(['first', 'second'],[], infix_operator='*')

class Range(Args):
    """
    Range generates an argument from a numerically interpolated range which
    is linear by default. An optional function can be specified to sample a
    numeric range with regular intervals.
    """

    key = param.String(default='', doc='''
         The key assigned to values computed over the linear numeric range.''')

    start_value =  param.Number(default=None, allow_None=True, constant=True, doc='''
         The starting numeric value of the linear interpolation.''')

    end_value = param.Number(default=None, allow_None=True, constant=True, doc='''
         The ending numeric value of the linear interpolation (inclusive).''')

    steps = param.Integer(default=2, constant=True, bounds=(2,None), doc='''
         The number of steps to interpolate over. Default is 2 which returns the
         start and end values without interpolation.''')

    # Can't this be a lambda?
    mapfn = param.Callable(default=identityfn, constant=True, doc='''
         The function to be mapped across the linear range. Identity  by default ''')

    def __init__(self, key, start_value, end_value, steps=2, mapfn=identityfn, **kwargs):

        values = self.linspace(start_value, end_value, steps)
        specs = [{key:mapfn(val)} for val in values ]

        super(Range, self).__init__(specs, key=key, start_value=start_value,
                                         end_value=end_value, steps=steps,
                                         mapfn=mapfn, **kwargs)
        self.pprint_args(['key', 'start_value'], ['end_value', 'steps'])

    def linspace(self, start, stop, n):
        """ Nice simple replacement for numpy linspace"""
        L = [0.0] * n
        nm1 = n - 1
        nm1inv = 1.0 / nm1
        for i in range(n):
            L[i] = nm1inv * (start*(nm1 - i) + stop*i)
        return L

class List(Args):
    """
    An argument specifier that takes its values from a given list.
    """

    list_values = param.List(default=[], constant=True, doc='''
         The list values that are to be returned by the specifier''')

    list_key = param.String(default='default', constant=True, doc='''
         The key assigned to the elements of the given list.''')

    def __init__(self, list_key, list_values, **kwargs):

        assert list_values != [], "Empty list not allowed."
        specs = [{list_key:val} for val in list_values]
        super(List, self).__init__(specs, list_key=list_key, list_values=list_values, **kwargs)
        self.pprint_args(['list_key', 'list_values'], [])

class Log(Args):
    """
    Specifier that loads arguments from a log file in tid (task id) order.  For
    full control over the arguments, you can use this class with Args as
    follows: Args(Log.extract_log(<log_file>).values()),

    This wrapper class allows a concise representation of log specifiers with
    the option of adding the task id to the loaded specifications.
    """

    log_path = param.String(default=None, allow_None=True, constant=True, doc='''
              The relative or absolute path to the log file. If a relative path
              is given, the absolute path is computed relative to os.getcwd().''')

    tid_key = param.String(default='tid', constant=True, allow_None=True, doc='''
               If not None, the key given to the tid values included in the
               loaded specifications. If None, the tid number is ignored.''')

    @staticmethod
    def extract_log(log_path, dict_type=dict):
        """
        Parses the log file generated by a launcher and returns dictionary with
        tid keys and specification values.

        Ordering can be maintained by setting dict_type to the appropriate
        constructor. Keys are converted from unicode to strings for kwarg use.
        """
        log_path = (log_path if os.path.isfile(log_path)
                    else os.path.join(os.getcwd(), log_path))
        with open(log_path,'r') as log:
            splits = (line.split() for line in log)
            uzipped = ((int(split[0]), json.loads(" ".join(split[1:]))) for split in splits)
            szipped = [(i, dict((str(k),v) for (k,v) in d.items())) for (i,d) in uzipped]
        return dict_type(szipped)

    @staticmethod
    def write_log(log_path, data, allow_append=True):
        """
        Writes the supplied specifications to the log path. The data may be
        supplied as either as a StaticSpecifier or as a list of dictionaries.

        By default, specifications will be appropriately appended to an existing
        log file. This can be disabled by setting allow_append to False.
        """
        append = os.path.isfile(log_path)
        listing = isinstance(data, list)

        if append and not allow_append:
            raise Exception('Appending has been disabled and file %s exists' % log_path)

        if not (listing or isinstance(data, Args)):
            raise Exception('Can only write static specifiers or dictionary lists to log file.')

        specs = data if listing else data.specs
        if not all(isinstance(el,dict) for el in specs):
            raise Exception('List elements must be dictionaries.')

        log_file = open(log_path, 'r+') if append else open(log_path, 'w')
        start = int(log_file.readlines()[-1].split()[0])+1 if append else 0
        ascending_indices = range(start, start+len(data))

        log_str = '\n'.join(['%d %s' % (tid, json.dumps(el)) for (tid, el) in zip(ascending_indices, specs)])<commit_msg>Parameter lacking constant=True and renamed listing variable to islist<commit_after>#
# Lancet core
#

import os, time, itertools, copy
import re, glob, string, re
import json

import param

try:
    import numpy as np
    np_ftypes = np.sctypes['float']
except:
    np, np_ftypes = None, []

try:    from pandas import DataFrame
except: DataFrame = None

from collections import defaultdict

float_types = [float] + np_ftypes
def identityfn(x): return x
def fp_repr(x):    return str(x) if (type(x) in float_types) else repr(x)

def set_fp_precision(value):
    """
    Function to set the floating precision across lancet.
    """
    BaseArgs.set_default('fp_precision', value)

#=====================#
# Argument Specifiers #
#=====================#

class BaseArgs(param.Parameterized):
    """
    The base class for all argument specifiers. Argument specifiers implement
    Python's iterator protocol, returning arguments on each call to next(). Note
    that these objects should not be written as generators as they often need to
    be pickled.
    """

    fp_precision = param.Integer(default=4, constant=True, doc='''
         The floating point precision to use for floating point values.  Unlike
         other basic Python types, floats need care with their representation as
         you only want to display up to the precision actually specified.''')

    def __init__(self, **params):
        self._pprint_args = ([],[],None,{})
        super(BaseArgs,self).__init__(**params)
        self.pprint_args([],['fp_precision', 'dynamic'])

    def __iter__(self): return self

    def __contains__(self, value):
        return value in (self.constant_keys + self.varying_keys)

    @classmethod
    def spec_formatter(cls, spec):
        " Formats the elements of an argument set appropriately"
        return dict((k, str(v)) for (k,v) in spec.items())

    @property
    def constant_keys(self):
        """
        Returns the list of parameter names whose values are constant as the
        argument specifier is iterated.  Note that the union of constant and
        varying_keys should partition the entire set of keys.
        """
        raise NotImplementedError

    @property
    def constant_items(self):
        """
        Returns the set of constant items as a list of tuples. This allows easy
        conversion to dictionary format. Note, the items should be supplied in
        the same key ordering as for constant_keys for consistency.
        """
        raise NotImplementedError

    @property
    def varying_keys(self):
        """
        Returns the list of parameters whose values vary as the argument
        specifier is iterated.  Whenever it is possible, keys should be sorted
        from those slowest to faster varying and sorted alphanumerically within
        groups that vary at the same rate.
        """
        raise NotImplementedError

    def round_floats(self, specs, fp_precision):
        _round_float = lambda v, fp: np.round(v, fp) if (type(v) in np_ftypes) else round(v, fp)
        _round = (lambda v, fp: int(v)) if fp_precision==0  else _round_float
        return (dict((k, _round(v, fp_precision) if (type(v) in float_types) else v)
                     for (k,v) in spec.items()) for spec in specs)

    def next(self):
        """
        Called to get a list of specifications: dictionaries with parameter name
        keys and string values.
        """
        raise StopIteration

    def copy(self):
        """
        Convenience method to avoid using the specifier without exhausting it.
        """
        return copy.copy(self)

    def _collect_by_key(self,specs):
        """
        Returns a dictionary like object with the lists of values collapsed by
        their respective key. Useful to find varying vs constant keys and to
        find how fast keys vary.
        """
        # Collects key, value tuples as list of lists then flatten using chain
        allkeys = itertools.chain.from_iterable([[(k, run[k])  for k in run] for run in specs])
        collection = defaultdict(list)
        for (k,v) in allkeys: collection[k].append(v)
        return collection

    def __add__(self, other):
        """
        Concatenates two argument specifiers.
        """
        if not other: return self
        return Concatenate(self,other)

    def __mul__(self, other):
        """
        Takes the Cartesian product of two argument specifiers.
        """
        return CartesianProduct(self, other)

    def __radd__(self, other):
        if not other: return self

    def __rmul__(self, other):
        if not other: return []

    def _cartesian_product(self, first_specs, second_specs):
        """
        Takes the Cartesian product of the specifications. Result will contain N
        specifications where N = len(first_specs) * len(second_specs) and keys
        are merged.  Example: [{'a':1},{'b':2}] * [{'c':3},{'d':4}] =
        [{'a':1,'c':3},{'a':1,'d':4},{'b':2,'c':3},{'b':2,'d':4}]
        """
        return  [ dict(zip(
                          list(s1.keys()) + list(s2.keys()),
                          list(s1.values()) + list(s2.values())
                      ))
                 for s1 in first_specs for s2 in second_specs ]

    def pprint_args(self, pos_args, keyword_args, infix_operator=None, extra_params={}):
        """
        Method to define the positional arguments and keyword order for pretty printing.
        """
        if infix_operator and not (len(pos_args)==2 and keyword_args==[]):
            raise Exception('Infix format requires exactly two positional arguments and no keywords')
        (kwargs,_,_,_) = self._pprint_args
        self._pprint_args = (keyword_args + kwargs, pos_args, infix_operator, extra_params)

    def _pprint(self, cycle=False, flat=False, annotate=False, level=1, tab = '   '):
        """
        Pretty printer that prints only the modified keywords and generates flat
        representations (for repr) and optionally annotates with a comment.
        """
        (kwargs, pos_args, infix_operator, extra_params) = self._pprint_args
        (br, indent)  = ('' if flat else '\n', '' if flat else tab * level)
        prettify = lambda x: isinstance(x, BaseArgs) and not flat
        pretty = lambda x: x._pprint(flat=flat, level=level+1) if prettify(x) else repr(x)

        params = dict(self.get_param_values())
        modified = [k for (k,v) in self.get_param_values(onlychanged=True)]
        pkwargs = [(k, params[k])  for k in kwargs if (k in modified)] + extra_params.items()
        arg_list = [(k,params[k]) for k in pos_args] + pkwargs

        len_ckeys, len_vkeys = len(self.constant_keys), len(self.varying_keys)
        info_triple = (len(self),
                       ', %d constant key(s)' % len_ckeys if len_ckeys else '',
                       ', %d varying key(s)'  % len_vkeys if len_vkeys else '')
        annotation = '# == %d items%s%s ==\n' % info_triple
        lines = [annotation] if annotate else []    # Optional annotating comment

        if cycle:
            lines.append('%s(...)' % self.__class__.__name__)
        elif infix_operator:
            level = level - 1
            triple = (pretty(params[pos_args[0]]), infix_operator, pretty(params[pos_args[1]]))
            lines.append('%s %s %s' % triple)
        else:
            lines.append('%s(' % self.__class__.__name__)
            for (k,v) in arg_list:
                lines.append('%s%s=%s' %  (br+indent, k, pretty(v)))
                lines.append(',')
            lines = lines[:-1] +[br+(tab*(level-1))+')'] # Remove trailing comma

        return ''.join(lines)

    def __repr__(self):  return self._pprint(flat=True)
    def __str__(self): return self._pprint()

class Args(BaseArgs):

    specs = param.List(default=[], constant=True, doc='''
          The static list of specifications (ie. dictionaries) to be
          returned by the specifier. Float values are rounded to
          fp_precision.''')

    def __init__(self, specs=None, fp_precision=None, **kwargs):
        if fp_precision is None: fp_precision = BaseArgs.fp_precision
        raw_specs, kwargs, explicit = self._build_specs(specs, kwargs, fp_precision)
        super(Args, self).__init__(fp_precision=fp_precision, specs=raw_specs, **kwargs)

        self._lexorder = None
        # Some types cannot be sorted (e.g. numpy arrays)
        self.unsortable_keys = []
        if explicit: 
            self.pprint_args(['specs'],[])
        else: # Present in kwarg format
            self.pprint_args([], self.constant_keys, None, 
                             dict(self.constant_items))

    def _build_specs(self, specs, kwargs, fp_precision):
        """
        Returns the specs, the remaining kwargs and whether or not the
        constructor was called with kwarg or explicit specs.
        """
        if isinstance(specs, list):
            return list(self.round_floats(specs, fp_precision)), kwargs, True
        elif specs is None:
            overrides = param.ParamOverrides(self, kwargs, allow_extra_keywords=True)
            extra_kwargs = overrides.extra_keywords()
            kwargs = dict([(k,v) for (k,v) in kwargs.items() if k not in extra_kwargs])
            return list(self.round_floats([extra_kwargs], fp_precision)), kwargs, False

    def __iter__(self):
        self._exhausted = False
        return self

    def next(self):
        if self._exhausted:
            raise StopIteration
        else:
            self._exhausted=True
            return self.specs

    def _unique(self, sequence, idfun=repr):
        """
        Note: repr() must be implemented properly on all objects. This is
        assumed by lancet when Python objects need to be formatted to string
        representation.
        """
        seen = {}
        return [seen.setdefault(idfun(e),e) for e in sequence
                if idfun(e) not in seen]

    def show(self):
        """
        Convenience method to inspect the available argument values in
        human-readable format. The ordering of keys is determined by
        how quickly they vary.
        """
        ordering = self.constant_keys + self.varying_keys
        spec_lines = [', '.join(['%s=%s' % (k, s[k]) for k in ordering if k in s]) for s in self.specs]
        print('\n'.join(['%d: %s' % (i,l) for (i,l) in enumerate(spec_lines)]))

    @property
    def constant_keys(self):
        collection = self._collect_by_key(self.specs)
        return [k for k in sorted(collection) if (len(self._unique(collection[k])) == 1)]

    @property
    def constant_items(self):
        collection = self._collect_by_key(self.specs)
        return [(k,collection[k][0]) for k in self.constant_keys]

    @property
    def varying_keys(self):
        collection = self._collect_by_key(self.specs)
        constant_set = set(self.constant_keys)
        unordered_varying = set(collection.keys()).difference(constant_set)
        # Finding out how fast keys are varying
        grouplens = [(len([len(list(y)) for (_,y) in itertools.groupby(collection[k])]),k) for k in collection]
        varying_counts = [(n,k) for (n,k) in sorted(grouplens) if (k in unordered_varying)]
        # Grouping keys with common frequency alphanumerically (desired behaviour).
        ddict = defaultdict(list)
        for (n,k) in varying_counts: ddict[n].append(k)
        alphagroups = [sorted(ddict[k]) for k in sorted(ddict)]
        return [el for group in alphagroups for el in group]

    @property
    def dframe(self):
        return DataFrame(self.specs) if DataFrame else "Pandas not available"

    def __len__(self): return len(self.specs)


class Concatenate(Args):
    """
    Concatenate is the sequential composition of two specifiers. The
    specifier created by the compositon (firsts + second) generates
    the arguments in first followed by the arguments in second.
    """

    first = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The first specifier used to generate the concatenation.''')

    second = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The second specifier used to generate the concatenation.''')

    def __init__(self, first, second):

        max_precision = max(first.fp_precision, second.fp_precision)
        specs = first.specs + second.specs
        super(Concatenate, self).__init__(specs, fp_precision=max_precision,
                                                first=first, second=second)
        self.pprint_args(['first', 'second'],[], infix_operator='+')

class CartesianProduct(Args):
    """
    CartesianProduct is the cartesian product of two specifiers. The
    specifier created by the compositon (firsts * second) generates
    the cartesian produce of the arguments in first followed by the
    arguments in second. Note that len(first * second) =
    len(first)*len(second)
    """

    first = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The first specifier used to generate the Cartesian product.''')

    second = param.ClassSelector(default=None, class_=Args, allow_None=True, constant=True, doc='''
            The second specifier used to generate the Cartesian product.''')

    def __init__(self, first, second):

        max_precision = max(first.fp_precision, second.fp_precision)
        specs = self._cartesian_product(first.specs, second.specs)

        overlap = (set(first.varying_keys + first.constant_keys)
                   &  set(second.varying_keys + second.constant_keys))
        assert overlap == set(), 'Sets of keys cannot overlap between argument specifiers in cartesian product.'
        super(CartesianProduct, self).__init__(specs, fp_precision=max_precision,
                                               first=first, second=second)
        self.pprint_args(['first', 'second'],[], infix_operator='*')

class Range(Args):
    """
    Range generates an argument from a numerically interpolated range which
    is linear by default. An optional function can be specified to sample a
    numeric range with regular intervals.
    """

    key = param.String(default='', constant=True, doc='''
         The key assigned to values computed over the linear numeric range.''')

    start_value =  param.Number(default=None, allow_None=True, constant=True, doc='''
         The starting numeric value of the linear interpolation.''')

    end_value = param.Number(default=None, allow_None=True, constant=True, doc='''
         The ending numeric value of the linear interpolation (inclusive).''')

    steps = param.Integer(default=2, constant=True, bounds=(2,None), doc='''
         The number of steps to interpolate over. Default is 2 which returns the
         start and end values without interpolation.''')

    # Can't this be a lambda?
    mapfn = param.Callable(default=identityfn, constant=True, doc='''
         The function to be mapped across the linear range. Identity  by default ''')

    def __init__(self, key, start_value, end_value, steps=2, mapfn=identityfn, **kwargs):

        values = self.linspace(start_value, end_value, steps)
        specs = [{key:mapfn(val)} for val in values ]

        super(Range, self).__init__(specs, key=key, start_value=start_value,
                                         end_value=end_value, steps=steps,
                                         mapfn=mapfn, **kwargs)
        self.pprint_args(['key', 'start_value'], ['end_value', 'steps'])

    def linspace(self, start, stop, n):
        """ Nice simple replacement for numpy linspace"""
        L = [0.0] * n
        nm1 = n - 1
        nm1inv = 1.0 / nm1
        for i in range(n):
            L[i] = nm1inv * (start*(nm1 - i) + stop*i)
        return L

class List(Args):
    """
    An argument specifier that takes its values from a given list.
    """

    list_values = param.List(default=[], constant=True, doc='''
         The list values that are to be returned by the specifier''')

    list_key = param.String(default='default', constant=True, doc='''
         The key assigned to the elements of the given list.''')

    def __init__(self, list_key, list_values, **kwargs):

        assert list_values != [], "Empty list not allowed."
        specs = [{list_key:val} for val in list_values]
        super(List, self).__init__(specs, list_key=list_key, list_values=list_values, **kwargs)
        self.pprint_args(['list_key', 'list_values'], [])

class Log(Args):
    """
    Specifier that loads arguments from a log file in tid (task id) order.  For
    full control over the arguments, you can use this class with Args as
    follows: Args(Log.extract_log(<log_file>).values()),

    This wrapper class allows a concise representation of log specifiers with
    the option of adding the task id to the loaded specifications.
    """

    log_path = param.String(default=None, allow_None=True, constant=True, doc='''
              The relative or absolute path to the log file. If a relative path
              is given, the absolute path is computed relative to os.getcwd().''')

    tid_key = param.String(default='tid', constant=True, allow_None=True, doc='''
               If not None, the key given to the tid values included in the
               loaded specifications. If None, the tid number is ignored.''')

    @staticmethod
    def extract_log(log_path, dict_type=dict):
        """
        Parses the log file generated by a launcher and returns dictionary with
        tid keys and specification values.

        Ordering can be maintained by setting dict_type to the appropriate
        constructor. Keys are converted from unicode to strings for kwarg use.
        """
        log_path = (log_path if os.path.isfile(log_path)
                    else os.path.join(os.getcwd(), log_path))
        with open(log_path,'r') as log:
            splits = (line.split() for line in log)
            uzipped = ((int(split[0]), json.loads(" ".join(split[1:]))) for split in splits)
            szipped = [(i, dict((str(k),v) for (k,v) in d.items())) for (i,d) in uzipped]
        return dict_type(szipped)

    @staticmethod
    def write_log(log_path, data, allow_append=True):
        """
        Writes the supplied specifications to the log path. The data may be
        supplied as either as a StaticSpecifier or as a list of dictionaries.

        By default, specifications will be appropriately appended to an existing
        log file. This can be disabled by setting allow_append to False.
        """
        append = os.path.isfile(log_path)
        islist = isinstance(data, list)

        if append and not allow_append:
            raise Exception('Appending has been disabled and file %s exists' % log_path)

        if not (islist or isinstance(data, Args)):
            raise Exception('Can only write static specifiers or dictionary lists to log file.')

        specs = data if islist else data.specs
        if not all(isinstance(el,dict) for el in specs):
            raise Exception('List elements must be dictionaries.')

        log_file = open(log_path, 'r+') if append else open(log_path, 'w')
        start = int(log_file.readlines()[-1].split()[0])+1 if append else 0
        ascending_indices = range(start, start+len(data))

        log_str = '\n'.join(['%d %s' % (tid, json.dumps(el)) for (tid, el) in zip(ascending_indices, specs)])
{'fraction_non_alphanumeric': 0.10646515100800742, 'fraction_numerical': 0.0025747315842323437, 'mean_word_length': 2.669690098261527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '55327', 'n_tokens_mistral': 11272, 'n_tokens_neox': 10886, 'n_words': 3662}