<commit_before>"""you are not expected to understand this implementation.
that's why it has tests.
the above-mentioned 'you' includes the author. :-}
"""

from __future__ import print_function, division

__version__ = '0.6.0'

import time
from functools import wraps
from contextlib import contextmanager
import inspect

# alias the builtins we shadow
_builtin_iter = iter

def print_metric(name, count, elapsed):
    """A metric function that prints

    :arg str name: name of the metric
    :arg int count: number of elements
    :arg float elapsed: time in seconds
    """
    if name is not None:
        print("%s: %d elements in %.2f seconds"%(name, count, elapsed))
    else:
        print("%d elements in %.2f seconds"%(count, elapsed))


default_metric = print_metric #: user-supplied function to use as global default metric

def call_default(name, count, elapsed):
    """call the :func:`default_metric` global in this module

    :arg str name: name of the metric
    :arg int count: number of elements
    :arg float elapsed: time in seconds
    """
    return default_metric(name, count, elapsed)


def make_multi_metric(*metrics):
    """Make a new metric function that calls the supplied metrics

    :arg functions metrics: metric functions
    :rtype: function
    """
    def multi_metric(name, count, elapsed):
        """Calls multiple metrics (closure)"""
        for m in metrics:
            m(name, count, elapsed)
    return multi_metric

def _do_iter(iterable, name, metric):
    total_time = 0
    count = 0
    it = enumerate(iterable, 1) # count, element
    try:
        while True:
            t = time.time()
            try:
                count, x = next(it)
            finally:
                total_time += time.time() - t
            yield x
    finally:
        # underlying iterable is exhausted (StopIteration) or errored. Record
        # the `metric` and allow exception to propogate
        metric(name, count, total_time)

def _do_each(iterable, name, metric):
    it = _builtin_iter(iterable)
    while True:
        t = time.time()
        try:
            x = next(it)
        except StopIteration:
            # don't record a metric for final next() call
            raise
        except Exception:
            # record a metric for other exceptions, than raise
            metric(name, 1, time.time() - t)
            raise
        else:
            # normal path, record metric and yield
            metric(name, 1, time.time() - t)
            yield x

def _do_first(iterable, name, metric):
    it = _builtin_iter(iterable)
    t = time.time()
    try:
        x = next(it)
    except StopIteration:
        # don't record a metric for final next() call
        raise
    except Exception:
        # record a metric for other exceptions, than raise
        metric(name, 1, time.time() - t)
        raise
    else:
        # normal path, record metric and yield
        metric(name, 1, time.time() - t)
        yield x

    for x in it: yield x

def _make_decorator(measuring_func):
    """morass of closures for making decorators/descriptors"""
    def _decorator(name = None, metric = call_default):
        def wrapper(func):

            name_ = name if name is not None else func.__module__ + '.' +func.__name__
            class instrument_decorator(object): # must be a class for descriptor magic to work
                @wraps(func)
                def __call__(self, *args, **kwargs):
                    return measuring_func(func(*args, **kwargs), name_, metric)

                def __get__(self, instance, class_):
                    name_ = name if name is not None else\
                        ".".join((class_.__module__, class_.__name__, func.__name__))
                    @wraps(func)
                    def wrapped_method(*args, **kwargs):
                        return measuring_func(func(instance, *args, **kwargs), name_, metric)
                    return wrapped_method
            return instrument_decorator()
<commit_msg>use yield from<commit_after>"""you are not expected to understand this implementation.
that's why it has tests.
the above-mentioned 'you' includes the author. :-}
"""

from __future__ import print_function, division

__version__ = '0.6.0'

import time
from functools import wraps
from contextlib import contextmanager
import inspect

# alias the builtins we shadow
_builtin_iter = iter

def print_metric(name, count, elapsed):
    """A metric function that prints

    :arg str name: name of the metric
    :arg int count: number of elements
    :arg float elapsed: time in seconds
    """
    if name is not None:
        print("%s: %d elements in %.2f seconds"%(name, count, elapsed))
    else:
        print("%d elements in %.2f seconds"%(count, elapsed))


default_metric = print_metric #: user-supplied function to use as global default metric

def call_default(name, count, elapsed):
    """call the :func:`default_metric` global in this module

    :arg str name: name of the metric
    :arg int count: number of elements
    :arg float elapsed: time in seconds
    """
    return default_metric(name, count, elapsed)


def make_multi_metric(*metrics):
    """Make a new metric function that calls the supplied metrics

    :arg functions metrics: metric functions
    :rtype: function
    """
    def multi_metric(name, count, elapsed):
        """Calls multiple metrics (closure)"""
        for m in metrics:
            m(name, count, elapsed)
    return multi_metric

def _do_iter(iterable, name, metric):
    total_time = 0
    count = 0
    it = enumerate(iterable, 1) # count, element
    try:
        while True:
            t = time.time()
            try:
                count, x = next(it)
            finally:
                total_time += time.time() - t
            yield x
    finally:
        # underlying iterable is exhausted (StopIteration) or errored. Record
        # the `metric` and allow exception to propogate
        metric(name, count, total_time)

def _do_each(iterable, name, metric):
    it = _builtin_iter(iterable)
    while True:
        t = time.time()
        try:
            x = next(it)
        except StopIteration:
            # don't record a metric for final next() call
            raise
        except Exception:
            # record a metric for other exceptions, than raise
            metric(name, 1, time.time() - t)
            raise
        else:
            # normal path, record metric and yield
            metric(name, 1, time.time() - t)
            yield x

def _do_first(iterable, name, metric):
    it = _builtin_iter(iterable)
    t = time.time()
    try:
        x = next(it)
    except StopIteration:
        # don't record a metric for final next() call
        raise
    except Exception:
        # record a metric for other exceptions, than raise
        metric(name, 1, time.time() - t)
        raise
    else:
        # normal path, record metric and yield
        metric(name, 1, time.time() - t)
        yield x

    yield from it

def _make_decorator(measuring_func):
    """morass of closures for making decorators/descriptors"""
    def _decorator(name = None, metric = call_default):
        def wrapper(func):

            name_ = name if name is not None else func.__module__ + '.' +func.__name__
            class instrument_decorator(object): # must be a class for descriptor magic to work
                @wraps(func)
                def __call__(self, *args, **kwargs):
                    return measuring_func(func(*args, **kwargs), name_, metric)

                def __get__(self, instance, class_):
                    name_ = name if name is not None else\
                        ".".join((class_.__module__, class_.__name__, func.__name__))
                    @wraps(func)
                    def wrapped_method(*args, **kwargs):
                        return measuring_func(func(instance, *args, **kwargs), name_, metric)
                    return wrapped_method
            return instrument_decorator()

{'fraction_non_alphanumeric': 0.08924113829256115, 'fraction_numerical': 0.0029955067398901645, 'mean_word_length': 2.189888535031847, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7211282', 'n_tokens_mistral': 2231, 'n_tokens_neox': 2152, 'n_words': 805}