<commit_before>from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals

import time


def _default_evaluate_until_callback(iter_num, res):
    print("Iter(%d): %s" % (iter_num, res))


# TODO move to handlers-specific module, since this assumes a handled_fn as
# input
def evaluate_until(fn,
                   gen,
                   max_iters=None,
                   max_seconds=None,
                   callback=_default_evaluate_until_callback):
    """
    evaluates a function on the output of a data generator until a given
    stopping condition

    fn:
    handled_fn
    """
    start_time = time.time()
    new_gen = enumerate(gen)
    try:
        while True:
            with fn.state.time("generating_data"):
                i, data = new_gen.next()
            if (max_iters is not None) and (i >= max_iters):
                break
            if ((max_seconds is not None)
                    and (time.time() - start_time >= max_seconds)):
                break
            res = fn(data)
            if callback is not None:
                callback(i, res)
    except StopIteration:
        pass<commit_msg>allow catching of C-c in evaluate_until<commit_after>from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals

import time


def _default_evaluate_until_callback(iter_num, res):
    print("Iter(%d): %s" % (iter_num, res))


# TODO move to handlers-specific module, since this assumes a handled_fn as
# input
def evaluate_until(fn,
                   gen,
                   max_iters=None,
                   max_seconds=None,
                   callback=_default_evaluate_until_callback,
                   catch_keyboard_interrupt=True):
    """
    evaluates a function on the output of a data generator until a given
    stopping condition

    fn:
    handled_fn
    """
    start_time = time.time()
    new_gen = enumerate(gen)

    to_catch = (StopIteration,)
    if catch_keyboard_interrupt:
        to_catch = to_catch + (KeyboardInterrupt,)

    print("Beginning evaluate_until")
    try:
        while True:
            with fn.state.time("generating_data"):
                i, data = new_gen.next()
            if (max_iters is not None) and (i >= max_iters):
                break
            if ((max_seconds is not None)
                    and (time.time() - start_time >= max_seconds)):
                break
            res = fn(data)
            if callback is not None:
                callback(i, res)
    except to_catch:
        print("Ending evaluate_until")
{'fraction_non_alphanumeric': 0.07863444572305332, 'fraction_numerical': 0.0, 'mean_word_length': 2.0185185185185186, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6494183', 'n_tokens_mistral': 743, 'n_tokens_neox': 712, 'n_words': 209}