<commit_before>## {{{ http://code.activestate.com/recipes/577504/ (r3) MIT licensed
from __future__ import absolute_import, division, print_function
from sys import getsizeof, stderr
from itertools import chain
from collections import deque
try:
    from reprlib import repr
except ImportError:
    pass


def total_size(o, handlers={}, verbose=False):
    """ Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,<commit_msg>[tools.memory] add memory_usage() and print_memory_usage()<commit_after>## {{{ http://code.activestate.com/recipes/577504/ (r3) MIT licensed
from __future__ import absolute_import, division, print_function
from sys import getsizeof, stderr
from itertools import chain
from collections import deque
import resource
try:
    from reprlib import repr
except ImportError:
    pass


last_memory_usage = 0
def memory_usage(unit='mb'):
    '''Returns the memory usage of the current process in bytes

    Returns
    -------
    usage
        Current memory usage.
    change
        Change of memory usage since last call.
    '''
    unit = unit.lower()
    assert unit in ('b', 'kb', 'mb', 'gb')
    factors = {'b': 1,
               'kb': 1024,
               'mb': 1024**2,
               'gb': 1024**3}
    global last_memory_usage
    x = last_memory_usage
    last_memory_usage = resource.getrusage(resource.RUSAGE_SELF)[2] * 1024
    return (last_memory_usage / factors[unit], (last_memory_usage - x) / factors[unit])

def print_memory_usage(msg=None, unit='mb'):
    u = memory_usage(unit)
    if msg is None:
        print('Memory usage {0:5.1f} {1} - delta: {2:5.1f} {1}'.format(u[0], unit.upper(), u[1]))
    else:
        print('Memory usage {0:5.1f} {1} - delta: {2:5.1f} {1} - {3}'.format(u[0], unit.upper(), u[1], msg))


def total_size(o, handlers={}, verbose=False):
    """ Returns the approximate memory footprint an object and all of its contents.

    Automatically finds the contents of the following builtin containers and
    their subclasses:  tuple, list, deque, dict, set and frozenset.
    To search other containers, add handlers to iterate over their contents:

        handlers = {SomeContainerClass: iter,
                    OtherContainerClass: OtherContainerClass.get_elements}

    """
    dict_handler = lambda d: chain.from_iterable(d.items())
    all_handlers = {tuple: iter,
                    list: iter,
                    deque: iter,
                    dict: dict_handler,
{'fraction_non_alphanumeric': 0.10281923714759536, 'fraction_numerical': 0.01857379767827529, 'mean_word_length': 2.9270833333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7359562', 'n_tokens_mistral': 912, 'n_tokens_neox': 854, 'n_words': 285}