<commit_before>from datetime import datetime, timedelta
import inspect

from .decorators import wraps
from .cross import PY2

__all__ = ['memoize', 'make_lookuper', 'silent_lookuper', 'cache']


# TODO: guard from keyword arguments in memoize, cache and make_lookuper.
#       For now it's complicated to do in python 2/3 compatible mode,
#       thanks getargspec/getfullargspec/signature mess.


class SkipMemoization(Exception):
    pass


def memoize(func):
    memory = {}

    @wraps(func)
    def wrapper(*args, **kwargs):
        if kwargs:
            key = args + tuple(sorted(kwargs.items()))
        else:
            key = args
        try:
            return memory[key]
        except KeyError:
            try:
                value = memory[key] = func(*args, **kwargs)
                return value
            except SkipMemoization as e:
                return e.args[0] if e.args else None
    return wrapper
memoize.skip = SkipMemoization


def _make_lookuper(silent):
    def make_lookuper(func):
        has_args, has_keys = has_arg_types(func)
        assert not has_keys, \
            'Lookup table building function should not have keyword arguments'

        if has_args:
            @memoize
            def wrapper(*args):
                f = lambda: func(*args)
                f.__name__ = '%s(%s)' % (func.__name__, ', '.join(map(str, args)))
                return make_lookuper(f)
        else:
            memory = {}

            def wrapper(arg):
                if not memory:
                    memory[object()] = None # prevent continuos memory refilling
                    memory.update(func())

                if silent:
                    return memory.get(arg)
                elif arg in memory:
                    return memory[arg]
                else:
                    raise LookupError("Failed to look up %s(%s)" % (func.__name__, arg))

        return wraps(func)(wrapper)
    return make_lookuper

make_lookuper = _make_lookuper(False)
silent_lookuper = _make_lookuper(True)


def cache(timeout):
    if isinstance(timeout, int):
        timeout = timedelta(seconds=timeout)

    def decorator(func):
        cache = {}

        @wraps(func)
        def wrapper(*args):
            if args in cache:
                result, timestamp = cache[args]
                if datetime.now() - timestamp < timeout:
                    return result
                else:
                    del cache[args]

            result = func(*args)
            cache[args] = result, datetime.now()
            return result

        def invalidate(*args):
            cache.pop(args)
        wrapper.invalidate = invalidate

        def invalidate_all():
            cache.clear()
        wrapper.invalidate_all = invalidate_all<commit_msg>Support **kwargs in @cache-d functions<commit_after>from datetime import datetime, timedelta
import inspect

from .decorators import wraps
from .cross import PY2

__all__ = ['memoize', 'make_lookuper', 'silent_lookuper', 'cache']


# TODO: guard from keyword arguments in memoize, cache and make_lookuper.
#       For now it's complicated to do in python 2/3 compatible mode,
#       thanks getargspec/getfullargspec/signature mess.


class SkipMemoization(Exception):
    pass


def memoize(func):
    memory = {}

    @wraps(func)
    def wrapper(*args, **kwargs):
        if kwargs:
            key = args + tuple(sorted(kwargs.items()))
        else:
            key = args
        try:
            return memory[key]
        except KeyError:
            try:
                value = memory[key] = func(*args, **kwargs)
                return value
            except SkipMemoization as e:
                return e.args[0] if e.args else None
    return wrapper
memoize.skip = SkipMemoization


def _make_lookuper(silent):
    def make_lookuper(func):
        has_args, has_keys = has_arg_types(func)
        assert not has_keys, \
            'Lookup table building function should not have keyword arguments'

        if has_args:
            @memoize
            def wrapper(*args):
                f = lambda: func(*args)
                f.__name__ = '%s(%s)' % (func.__name__, ', '.join(map(str, args)))
                return make_lookuper(f)
        else:
            memory = {}

            def wrapper(arg):
                if not memory:
                    memory[object()] = None # prevent continuos memory refilling
                    memory.update(func())

                if silent:
                    return memory.get(arg)
                elif arg in memory:
                    return memory[arg]
                else:
                    raise LookupError("Failed to look up %s(%s)" % (func.__name__, arg))

        return wraps(func)(wrapper)
    return make_lookuper

make_lookuper = _make_lookuper(False)
silent_lookuper = _make_lookuper(True)


def cache(timeout):
    if isinstance(timeout, int):
        timeout = timedelta(seconds=timeout)

    def decorator(func):
        cache = {}

        @wraps(func)
        def wrapper(*args, **kwargs):
            if kwargs:
                key = args + tuple(sorted(kwargs.items()))
            else:
                key = args

            if key in cache:
                result, timestamp = cache[key]
                if datetime.now() - timestamp < timeout:
                    return result
                else:
                    del cache[key]

            result = func(*key, **kwargs)
            cache[key] = result, datetime.now()
            return result

        def invalidate(*args, **kwargs):
            cache.pop(args + tuple(sorted(kwargs.items())))
        wrapper.invalidate = invalidate

        def invalidate_all():
            cache.clear()
        wrapper.invalidate_all = invalidate_all
{'fraction_non_alphanumeric': 0.09227839335180055, 'fraction_numerical': 0.0013850415512465374, 'mean_word_length': 1.7366177167219328, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860276', 'n_tokens_mistral': 1586, 'n_tokens_neox': 1509, 'n_words': 433}