<commit_before># the caching decorator for helpers functions

class NotFound: pass

class Debug_MasterCache(dict):
    def get(self, newKey, default):
        ret = dict.get(self, newKey, NotFound)
        if ret is not NotFound:
            print("usek: "+format(newKey, '#067b'))
            if Cache.masterCache[newKey]['key'] != newKey:
                print("ERROR: key stored in cache is not the same as the new key !!!!")
                print("      "+format(self[newKey]['key'], '#067b')+" key in cache")
            return ret
        else:
            return default

    def __setitem__(self, newKey, value):
        dict.__setitem__(self, newKey, value)
        if newKey != 0:
            print("newk: "+format(newKey, '#067b'))

class Debug_Cache(dict):
    def get(self, name, default):
        ret = dict.get(self, name, NotFound)
        if ret is not NotFound:
            if name in Cache.funcs:
                print("cache found for {}: {}".format(name, self[name]))
            return ret
        else:
            return default

    def __setitem__(self, name, value):
        dict.__setitem__(self, name, value)
        if name != 'key' and name in Cache.funcs:
            print("cache added for {}: {}".format(name, self[name]))

    def validate(self, name, ret):
        if ret != self[name]:
            print("ERROR: cache ({}) != current ({}) for {}".format(self[name], ret, name))

class Cache:
    cache = {}
    masterCache = {}
    funcs = set()
    lambdas = set()
    debug = False

    @staticmethod
    def reset():
        # reinit the whole cache
        key = 0

        if Cache.debug:
            Cache.masterCache = Debug_MasterCache()
            Cache.cache = Debug_Cache()
            Cache.cache['key'] = key
        else:
            Cache.masterCache = { }
            Cache.cache = { }

        Cache.masterCache[key] = Cache.cache

    @staticmethod
    def update(newKey):
        cache = Cache.masterCache.get(newKey, None)
        if cache is None:
            cache = Debug_Cache(key=newKey) if Cache.debug else { }
            Cache.cache = cache
            Cache.masterCache[newKey] = cache
        else:
            Cache.cache = cache

    @staticmethod
    def decorator(func):
        name = func.__name__
        Cache.funcs.add(name)
        return Cache._decorate(name, func)

    # for lambdas
    @staticmethod
    def ldeco(name, func):
        Cache.lambdas.add(name)
        return Cache._decorate(name, func)

    @staticmethod
    def _decorate(name, func):
        def _decorator(self):
            ret = Cache.cache.get(name, None)
            if ret is not None:
                if Cache.debug and name in Cache.funcs:
                    ret = func(self)
                    Cache.cache.validate(name, ret)
                return ret
            else:
                ret = func(self)
                Cache.cache[name] = ret
                return ret
        return _decorator

class RequestCache(object):
    def __init__(self):
        self.results = {}

    def request(self, request, *args):
        return ''.join([request] + [str(arg) for arg in args])

    def store(self, request, result):
        self.results[request] = result

    def get(self, request):<commit_msg>cache: Make Cache an instance<commit_after># the caching decorator for helpers functions

class NotFound: pass

class Debug_MasterCache(dict):
    def get(self, newKey, default):
        ret = dict.get(self, newKey, NotFound)
        if ret is not NotFound:
            print("usek: "+format(newKey, '#067b'))
            if Cache.masterCache[newKey]['key'] != newKey:
                print("ERROR: key stored in cache is not the same as the new key !!!!")
                print("      "+format(self[newKey]['key'], '#067b')+" key in cache")
            return ret
        else:
            return default

    def __setitem__(self, newKey, value):
        dict.__setitem__(self, newKey, value)
        if newKey != 0:
            print("newk: "+format(newKey, '#067b'))

class Debug_Cache(dict):
    def get(self, name, default):
        ret = dict.get(self, name, NotFound)
        if ret is not NotFound:
            if name in Cache.funcs:
                print("cache found for {}: {}".format(name, self[name]))
            return ret
        else:
            return default

    def __setitem__(self, name, value):
        dict.__setitem__(self, name, value)
        if name != 'key' and name in Cache.funcs:
            print("cache added for {}: {}".format(name, self[name]))

    def validate(self, name, ret):
        if ret != self[name]:
            print("ERROR: cache ({}) != current ({}) for {}".format(self[name], ret, name))

class VersionedCache(object):
    def __init__(self):
        self.cache = {}
        self.masterCache = {}
        self.funcs = set()
        self.lambdas = set()
        self.debug = False

    def reset(self):
        # reinit the whole cache
        key = 0

        if self.debug:
            self.masterCache = Debug_MasterCache()
            self.cache = Debug_Cache()
            self.cache['key'] = key
        else:
            self.masterCache = { }
            self.cache = { }

        self.masterCache[key] = self.cache

    def update(self, newKey):
        cache = self.masterCache.get(newKey, None)
        if cache is None:
            cache = Debug_Cache(key=newKey) if self.debug else { }
            self.cache = cache
            self.masterCache[newKey] = cache
        else:
            self.cache = cache

    def decorator(self, func):
        name = func.__name__
        self.funcs.add(name)
        return self._decorate(name, func)

    # for lambdas
    def ldeco(self, name, func):
        self.lambdas.add(name)
        return self._decorate(name, func)

    def _decorate(self, name, func):
        def _decorator(arg):
            ret = self.cache.get(name, None)
            if ret is not None:
                if self.debug and name in self.funcs:
                    ret = func(arg)
                    self.cache.validate(name, ret)
                return ret
            else:
                ret = func(arg)
                self.cache[name] = ret
                return ret
        return _decorator

Cache = VersionedCache()

class RequestCache(object):
    def __init__(self):
        self.results = {}

    def request(self, request, *args):
        return ''.join([request] + [str(arg) for arg in args])

    def store(self, request, result):
        self.results[request] = result

    def get(self, request):
{'fraction_non_alphanumeric': 0.1163641909409791, 'fraction_numerical': 0.0033551929235931064, 'mean_word_length': 1.857516339869281, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3411564', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1856, 'n_words': 532}