<commit_before>#
# Experimental code (WIP)
#

import os, sys, json, time, fnmatch, pipes, subprocess
import param
from core import BaseArgs, Concatenate, CartesianProduct

class DynamicArgs(BaseArgs):

    output_extractor = param.Callable(default=json.loads, doc="""
       The function that returns the relevant data from the standard
       output stream dumped to file in the streams subdirectory. This
       information must be retyrned in a format suitable for updating
       the specifier.. By default uses json.loads but pickle.loads
       could also be a valid option. The callable must take a string
       and return a python object suitable for updating the specifier.""")


    def __init__(self, **kwargs):
        super(DynamicArgs, self).__init__(**kwargs)
        # Returned next iteration and updated by _updated_state method
        self._next_val = self._initial_state(**kwargs)


    def _update_state(self, data):
        """
        Determines the next desired point in the parameter space using
        the parsed data returned from the public update
        method. Returns the value that will next emitted on the next
        iteration using the list of dictionaries format for
        arguments. If the update fails or data is None, StopIteration
        should be supplied as the return value.
        """
        raise NotImplementedError


    def _initial_state(self, **kwargs):
        """
        Reset the the DynamicArgs object to its initial state and used
        to reset the object adter the iterator is exhausted. The
        return value is the initial argument to be returned by next().
        """
        raise NotImplementedError


    def next(self):
        if self._next_val is StopIteration:
            self._initial_state()
            raise StopIteration
        current_val =  self._next_val
        self._next_val = StopIteration
        return current_val


    def update(self, tids, info):
        """
        Called to update the state of the iterator.  This methods
        receives the set of task ids from the previous set of tasks
        together with the launch information to allow the output
        values to be parsed using the output_extractor. This data is then
        used to determine the next desired point in the parameter
        space by calling the _update_state method.
        """
        outputs_dir = os.path.join(info['root_directory'], 'streams')
        pattern = '%s_*_tid_*{tid}.o.{tid}*' % info['batch_name']
        flist = os.listdir(outputs_dir)
        try:
            outputs = []
            for tid in tids:
                matches = fnmatch.filter(flist, pattern.format(tid=tid))
                if len(matches) != 1:
                    self.warning("No unique output file for tid %d" % tid)
                contents = open(os.path.join(outputs_dir, matches[0]),'r').read()
                outputs.append(self.output_extractor(contents))

            self._next_val = self._update_state(outputs)
        except:
            self.warning("Cannot load required output files. Cannot continue.")
            return None # StopIteration should be raised by the argument specifier


    def show(self):
        """
        When dynamic, not all argument values may be available.
        """
        copied = self.copy()
        enumerated = [el for el in enumerate(copied)]
        for (group_ind, specs) in enumerated:
            if len(enumerated) > 1: print("Group %d" % group_ind)
            ordering = self.constant_keys + self.varying_keys
            # Ordered nicely by varying_keys definition.
            spec_lines = [', '.join(['%s=%s' % (k, s[k]) for k in ordering]) for s in specs]
            print('\n'.join(['%d: %s' % (i,l) for (i,l) in enumerate(spec_lines)]))

        print('Remaining arguments not available for %s' % self.__class__.__name__)


    def __add__(self, other):
        """
        Concatenates two argument specifiers. See Concatenate and
        DynamicConcatenate documentation respectively.
        """
        if not other: return self
        dynamic = (isinstance(self, DynamicArgs),  isinstance(other, DynamicArgs))
        if dynamic == (True, True):
            raise Exception('Cannot concatenate two dynamic specifiers.')
        elif (True in dynamic):
            return DynamicConcatenate(self,other)
        else:
            return Concatenate(self,other)


    def __mul__(self, other):
        """
        Takes the cartesian product of two argument specifiers. See
        CartesianProduct and DynamicCartesianProduct documentation.
        """
        if not other: return []
        dynamic = (isinstance(self, DynamicArgs),  isinstance(other, DynamicArgs))
        if dynamic == (True, True):
            raise Exception('Cannot take Cartesian product two dynamic specifiers.')
        elif (True in dynamic):
            return DynamicCartesianProduct(self, other)
        else:
            return CartesianProduct(self, other)


#=============================#
# Dynamic argument specifiers #
#=============================#


class DynamicConcatenate(DynamicArgs):
    def __init__(self, first, second):
        self.first = first
        self.second = second
        super(Concatenate, self).__init__(dynamic=True)

        self._exhausted = False
        self._first_sent = False
        self._first_cached = None
        self._second_cached = None
        if not isinstance(first,DynamicArgs):
            self._first_cached = next(first.copy())<commit_msg>Introduced SimpleOptimization as an example of a Dynamic Args<commit_after>#
# Work in progress. Only SimpleOptimization is currently available.
#

import os, sys, json, time, fnmatch, pipes, subprocess
import param
from core import BaseArgs, Concatenate, CartesianProduct

class DynamicArgs(BaseArgs):

    output_extractor = param.Callable(default=json.loads, doc="""
       The function that returns the relevant data from the standard
       output stream dumped to file in the streams subdirectory. This
       information must be retyrned in a format suitable for updating
       the specifier.. By default uses json.loads but pickle.loads
       could also be a valid option. The callable must take a string
       and return a python object suitable for updating the specifier.""")


    def __init__(self, **kwargs):
        super(DynamicArgs, self).__init__(**kwargs)
        # Returned next iteration and updated by _updated_state method
        self._next_val = self._initial_state(**kwargs)


    def _update_state(self, data):
        """
        Determines the next desired point in the parameter space using
        the parsed data returned from the public update
        method. Returns the value that will next emitted on the next
        iteration using the list of dictionaries format for
        arguments. If the update fails or data is None, StopIteration
        should be supplied as the return value.
        """
        raise NotImplementedError


    def _initial_state(self, **kwargs):
        """
        Reset the the DynamicArgs object to its initial state and used
        to reset the object adter the iterator is exhausted. The
        return value is the initial argument to be returned by next().
        """
        raise NotImplementedError


    def next(self):
        if self._next_val is StopIteration:
            self._initial_state()
            raise StopIteration
        current_val =  self._next_val
        self._next_val = StopIteration
        return current_val


    def update(self, tids, info):
        """
        Called to update the state of the iterator.  This methods
        receives the set of task ids from the previous set of tasks
        together with the launch information to allow the output
        values to be parsed using the output_extractor. This data is then
        used to determine the next desired point in the parameter
        space by calling the _update_state method.
        """
        outputs_dir = os.path.join(info['root_directory'], 'streams')
        pattern = '%s_*_tid_*{tid}.o.{tid}*' % info['batch_name']
        flist = os.listdir(outputs_dir)
        try:
            outputs = []
            for tid in tids:
                matches = fnmatch.filter(flist, pattern.format(tid=tid))
                if len(matches) != 1:
                    self.warning("No unique output file for tid %d" % tid)
                contents = open(os.path.join(outputs_dir, matches[0]),'r').read()
                outputs.append(self.output_extractor(contents))

            self._next_val = self._update_state(outputs)
        except:
            self.warning("Cannot load required output files. Cannot continue.")
            return None # StopIteration should be raised by the argument specifier


    def show(self):
        """
        When dynamic, not all argument values may be available.
        """
        copied = self.copy()
        enumerated = [el for el in enumerate(copied)]
        for (group_ind, specs) in enumerated:
            if len(enumerated) > 1: print("Group %d" % group_ind)
            ordering = self.constant_keys + self.varying_keys
            # Ordered nicely by varying_keys definition.
            spec_lines = [', '.join(['%s=%s' % (k, s[k]) for k in ordering]) for s in specs]
            print('\n'.join(['%d: %s' % (i,l) for (i,l) in enumerate(spec_lines)]))

        print('Remaining arguments not available for %s' % self.__class__.__name__)


    def __add__(self, other):
        """
        Concatenates two argument specifiers. See Concatenate and
        DynamicConcatenate documentation respectively.
        """
        if not other: return self
        dynamic = (isinstance(self, DynamicArgs),  isinstance(other, DynamicArgs))
        if dynamic == (True, True):
            raise Exception('Cannot concatenate two dynamic specifiers.')
        elif (True in dynamic):
            return DynamicConcatenate(self,other)
        else:
            return Concatenate(self,other)


    def __mul__(self, other):
        """
        Takes the cartesian product of two argument specifiers. See
        CartesianProduct and DynamicCartesianProduct documentation.
        """
        if not other: return []
        dynamic = (isinstance(self, DynamicArgs),  isinstance(other, DynamicArgs))
        if dynamic == (True, True):
            raise Exception('Cannot take Cartesian product two dynamic specifiers.')
        elif (True in dynamic):
            return DynamicCartesianProduct(self, other)
        else:
            return CartesianProduct(self, other)



class SimpleOptimization(DynamicArgs):
    """
    Very simple optimizer designed to illustrate how Dynamic Args may
    be implemented. The implementation uses is chosen for simplicity
    of understanding and not for efficiency. For instance, it could be
    made much more efficient by caching the values of previously
    explored states.

    It works by greedily maximizing or minimizing some value via
    greedy gradient ascent/descent. The local parameter space is
    explored by examining the change in output value when an increment
    or decrement of 'stepsize' is made in the parameter space from the
    current position, initialized at the 'start' value. The
    optimization terminates when either a local minima/maxima has been
    found or when 'max_steps' is reached.
    """

    key = param.String(constant=True, doc="""
        The name of the argument that will be optimized in a greedy fashion.""")

    mode = param.ObjectSelector(default='minimum', objects=['minimum', 'maximum'], constant=True,
       doc="""Whether a local minimum or maximum is sought.""")

    start = param.Number(default=0.0, constant=True, doc="""
        The starting argument value for the gradient ascent or descent""")

    stepsize = param.Number(default=1.0, constant=True, doc="""
        The size of the steps taken in parameter space.""")

    max_steps=param.Integer(default=100, constant=True, doc="""
        Once max_steps is reached, the optimization terminates.""")

    def __init__(self, key, **kwargs):
        super(SimpleOptimization, self).__init__(key=key, **kwargs)
        self.pprint_args(['key', 'start', 'stepsize'],[])

    def _initial_state(self, **kwargs):
        self._steps_complete = 0
        self._best_val = float('inf')
        self._arg = self.start
        return [{self.key:self.start+1}, {self.key:self.start-1}]

    def _update_state(self, vals):
        """
        Takes as input a list or tuple of two elements. First the
        value returned by incrementing by 'stepsize' followed by the
        value returned after a 'stepsize' decrement.
        """
        self._steps_complete += 1
        if self._steps_complete == self.max_steps:
            self.warning('Maximum step limit of %d reached.' % self.max_steps)
            return StopIteration

        arg_inc, arg_dec = vals
        evaluator = max if self.mode == 'maximize' else min
        best_val = evaluator(arg_inc, arg_dec, self._best_val)
        if best_val == self._best_val:
            info = (self.mode.capitalize(), best_val, self._arg)
            self.message("%s value '%r' found at argmin '%r'"  % info)
            return StopIteration

        increment = (self.mode=='minimum') and (arg_dec > arg_inc)
        self._arg += self.stepsize if increment else -self.stepsize
        self._best_val= best_val
        return [{self.key:self._arg+self.stepsize},
                {self.key:self._arg-self.stepsize}]

    @property
    def constant_keys(self):  return []
    @property
    def constant_items(self): return []
    @property
    def varying_keys(self):   return [self.key]

    def __len__(self): return self.max_steps


#=========================#
# Experimental code (WIP) #
#=========================#


class DynamicConcatenate(DynamicArgs):
    def __init__(self, first, second):
        self.first = first
        self.second = second
        super(Concatenate, self).__init__(dynamic=True)

        self._exhausted = False
        self._first_sent = False
        self._first_cached = None
        self._second_cached = None
        if not isinstance(first,DynamicArgs):
            self._first_cached = next(first.copy())
{'fraction_non_alphanumeric': 0.09163514081546868, 'fraction_numerical': 0.0011909766008126664, 'mean_word_length': 2.546583850931677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3552837', 'n_tokens_mistral': 3799, 'n_tokens_neox': 3661, 'n_words': 1332}