<commit_before>from ..core.parameterization.parameter_core import Observable
import itertools

class Cacher(object):
    """




    """

    def __init__(self, operation, limit=5, ignore_args=(), force_kwargs=()):
        self.limit = int(limit)
        self.ignore_args = ignore_args
        self.force_kwargs = force_kwargs
        self.operation=operation
        self.cached_inputs = []
        self.cached_outputs = []
        self.inputs_changed = []

    def __call__(self, *args, **kw):
        """
        A wrapper function for self.operation,
        """

        #ensure that specified arguments are ignored
        items = sorted(kw.items(), key=lambda x: x[0])
        oa_all = args + tuple(a for _,a in items)
        if len(self.ignore_args) != 0:
            oa = [a for i,a in itertools.chain(enumerate(args), items) if i not in self.ignore_args and i not in self.force_kwargs]
        else:
            oa = oa_all

        # this makes sure we only add an observer once, and that None can be in args
        observable_args = []
        for a in oa:
            if (not any(a is ai for ai in observable_args)) and a is not None:
                observable_args.append(a)

        #make sure that all the found argument really are observable:
        #otherswise don't cache anything, pass args straight though
        if not all([isinstance(arg, Observable) for arg in observable_args]):
            return self.operation(*args, **kw)

        if len(self.force_kwargs) != 0:
            # check if there are force args, which force reloading
            for k in self.force_kwargs:
                if k in kw and kw[k] is not None:
                    return self.operation(*args, **kw)
        # TODO: WARNING !!! Cache OFFSWITCH !!! WARNING
        # return self.operation(*args, **kw)

        #if the result is cached, return the cached computation
        state = [all(a is b for a, b in itertools.izip_longest(args, cached_i)) for cached_i in self.cached_inputs]
        try:
            if any(state):
                i = state.index(True)
                if self.inputs_changed[i]:
                    #(elements of) the args have changed since we last computed: update
                    self.cached_outputs[i] = self.operation(*args, **kw)
                    self.inputs_changed[i] = False
                return self.cached_outputs[i]
            else:
                #first time we've seen these arguments: compute

                #first make sure the depth limit isn't exceeded
                if len(self.cached_inputs) == self.limit:
                    args_ = self.cached_inputs.pop(0)
                    [a.remove_observer(self, self.on_cache_changed) for a in args_ if a is not None]
                    self.inputs_changed.pop(0)
                    self.cached_outputs.pop(0)
                #compute<commit_msg>bug fix: caching.py w.r.t. ignore_args<commit_after>from ..core.parameterization.parameter_core import Observable
import itertools

class Cacher(object):
    """




    """

    def __init__(self, operation, limit=5, ignore_args=(), force_kwargs=()):
        self.limit = int(limit)
        self.ignore_args = ignore_args
        self.force_kwargs = force_kwargs
        self.operation=operation
        self.cached_inputs = []
        self.cached_outputs = []
        self.inputs_changed = []

    def __call__(self, *args, **kw):
        """
        A wrapper function for self.operation,
        """

        #ensure that specified arguments are ignored
        items = sorted(kw.items(), key=lambda x: x[0])
        oa_all = args + tuple(a for _,a in items)
        if len(self.ignore_args) != 0:
            oa = [a for i,a in itertools.chain(enumerate(args), items) if i not in self.ignore_args and i not in self.force_kwargs]
        else:
            oa = oa_all

        # this makes sure we only add an observer once, and that None can be in args
        observable_args = []
        for a in oa:
            if (not any(a is ai for ai in observable_args)) and a is not None:
                observable_args.append(a)

        #make sure that all the found argument really are observable:
        #otherswise don't cache anything, pass args straight though
        if not all([isinstance(arg, Observable) for arg in observable_args]):
            return self.operation(*args, **kw)

        if len(self.force_kwargs) != 0:
            # check if there are force args, which force reloading
            for k in self.force_kwargs:
                if k in kw and kw[k] is not None:
                    return self.operation(*args, **kw)
        # TODO: WARNING !!! Cache OFFSWITCH !!! WARNING
        # return self.operation(*args, **kw)

        #if the result is cached, return the cached computation
        state = [all(a is b for a, b in itertools.izip_longest(args, cached_i)) for cached_i in self.cached_inputs]
        try:
            if any(state):
                i = state.index(True)
                if self.inputs_changed[i]:
                    #(elements of) the args have changed since we last computed: update
                    self.cached_outputs[i] = self.operation(*args, **kw)
                    self.inputs_changed[i] = False
                return self.cached_outputs[i]
            else:
                #first time we've seen these arguments: compute

                #first make sure the depth limit isn't exceeded
                if len(self.cached_inputs) == self.limit:
                    args_ = self.cached_inputs.pop(0)
                    args_ = [a for i,a in enumerate(args_) if i not in self.ignore_args and i not in self.force_kwargs]
                    [a.remove_observer(self, self.on_cache_changed) for a in args_ if a is not None]
                    self.inputs_changed.pop(0)
                    self.cached_outputs.pop(0)
                #compute
{'fraction_non_alphanumeric': 0.08951406649616368, 'fraction_numerical': 0.0023870417732310316, 'mean_word_length': 1.93006993006993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3904825', 'n_tokens_mistral': 1661, 'n_tokens_neox': 1579, 'n_words': 535}