<commit_before>import functools


class memoized(object):
    """Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """
    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):

        targs = (e for e in args)
        hh = tuple(  hashable(e) for e in targs )
        h = hash(hh)
        try:
            return self.cache[h]
        except KeyError:
            value = self.func(*args)
            self.cache[h] = value
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)
    def __repr__(self):
        """Return the function's docstring."""
        return self.func.__doc__
    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)


class cachedondisk(object):
    """Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """

    def __init__(self, func):

        # create caching direcory if it is not there already
        import os
        if not os.path.isdir('.cache'):
            os.mkdir('.cache')

        self.func = func
        self.fname = func.__name__

    def __call__(self, *args):
        import pickle
        hh = tuple(  hashable(e) for e in args )
        h = hash(hh)
        try:
            with file('.cache/{0}.{1}.pickle'.format(self.fname,h)) as f:
                value = pickle.load(f)
            return value
        except IOError:
            value = self.func(*args)
            if value is not None:  # should there be other kinds of error values
                # write file with h
                with file('.cache/{0}.{1}.pickle'.format(self.fname,h),'w') as f:
                    pickle.dump(value,f)
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)
    def __repr__(self):
        """Return the function's docstring."""
        return self.func.__doc__
    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)

def clear_cache(function_name=None):

    import os

    try:
        if function_name:
            os.system('rm -rf .cache/{}.*.pickle'.format(function_name))
        else:
            os.system('rm -rf .cache/*.pickle')
    except:
        pass

import os

class DiskDictionary:

    def __init__(self,directory='.cache',funname='fun'):
        self.funname = funname
        if not os.path.isdir(directory):
            os.mkdir(directory)
        self.directory = directory

    def get_filename(self, key):
        import pickle
        hh = tuple(hashable(k) for k in key)
        h = hash(hh)
        filename = '{0}/{1}.{2}.pickle'.format(self.directory,self.funname,h)
        return filename

    def __setitem__(self, key, value):
        import pickle
        filename = self.get_filename(key)
        try:
            with file(filename,'w') as f:
                pickle.dump(value,f) 
        except TypeError as e:
            raise e
            
    def get(self, item):
        import pickle
        filename = self.get_filename(item)  
        try:
            with file(filename) as f:
                value = pickle.load(f)
                return value
        except IOError:
            return None
    


import collections


def hashable(obj):
    if hasattr(obj,'flatten'): # for numpy arrays
        return tuple( obj.flatten().tolist() )
    if isinstance(obj, collections.Hashable):
        return obj
    if isinstance(obj, collections.Mapping):
        items = [(k,hashable(v)) for (k,v) in obj.items()]
        return frozenset(items)
    if isinstance(obj, collections.Iterable):
        return tuple([hashable(item) for item in obj])
    return TypeError(type(obj))<commit_msg>ENH: very unsophisticated cached on disk dictionary object<commit_after>import functools


class memoized(object):
    """Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """
    def __init__(self, func):
        self.func = func
        self.cache = {}

    def __call__(self, *args):

        targs = (e for e in args)
        hh = tuple(  hashable(e) for e in targs )
        h = hash(hh)
        try:
            return self.cache[h]
        except KeyError:
            value = self.func(*args)
            self.cache[h] = value
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)
    def __repr__(self):
        """Return the function's docstring."""
        return self.func.__doc__
    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)


class cachedondisk(object):
    """Decorator that caches a function's return value each time it is called.
    If called later with the same arguments, the cached value is returned, and
    not re-evaluated.
    """

    def __init__(self, func):

        # create caching direcory if it is not there already
        import os
        if not os.path.isdir('.cache'):
            os.mkdir('.cache')

        self.func = func
        self.fname = func.__name__

    def __call__(self, *args):
        import pickle
        hh = tuple(  hashable(e) for e in args )
        h = hash(hh)
        try:
            with file('.cache/{0}.{1}.pickle'.format(self.fname,h)) as f:
                value = pickle.load(f)
            return value
        except IOError:
            value = self.func(*args)
            if value is not None:  # should there be other kinds of error values
                # write file with h
                with file('.cache/{0}.{1}.pickle'.format(self.fname,h),'w') as f:
                    pickle.dump(value,f)
            return value
        except TypeError:
            # uncachable -- for instance, passing a list as an argument.
            # Better to not cache than to blow up entirely.
            return self.func(*args)
    def __repr__(self):
        """Return the function's docstring."""
        return self.func.__doc__
    def __get__(self, obj, objtype):
        """Support instance methods."""
        return functools.partial(self.__call__, obj)

def clear_cache(function_name=None):

    import os

    try:
        if function_name:
            os.system('rm -rf .cache/{}.*.pickle'.format(function_name))
        else:
            os.system('rm -rf .cache/*.pickle')
    except:
        pass

import os

class DiskDictionary:

    def __init__(self,directory='.cache',funname='fun'):
        self.funname = funname
        if not os.path.isdir(directory):
            os.mkdir(directory)
        self.directory = directory

    def get_filename(self, key):
        import pickle
        hh = tuple(hashable(k) for k in key)
        h = hash(hh)
        filename = '{0}/{1}.{2}.pickle'.format(self.directory,self.funname,h)
        return filename

    def __setitem__(self, key, value):
        import pickle
        filename = self.get_filename(key)
        try:
            with file(filename,'w') as f:
                pickle.dump(value,f) 
        except TypeError as e:
            raise e
            
    def get(self, item):
        import pickle
        filename = self.get_filename(item)  
        try:
            with file(filename) as f:
                value = pickle.load(f)
                return value
        except :
            return None
    


import collections


def hashable(obj):
    if hasattr(obj,'flatten'): # for numpy arrays
        return tuple( obj.flatten().tolist() )
    if isinstance(obj, collections.Hashable):
        return obj
    if isinstance(obj, collections.Mapping):
        items = [(k,hashable(v)) for (k,v) in obj.items()]
        return frozenset(items)
    if isinstance(obj, collections.Iterable):
        return tuple([hashable(item) for item in obj])
    return TypeError(type(obj))
{'fraction_non_alphanumeric': 0.10240541802895843, 'fraction_numerical': 0.0016347501167678655, 'mean_word_length': 1.9272043745727956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3907491', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2390, 'n_words': 753}