<commit_before>from collections import namedtuple
from functools import partial


DONE = 0
QUERY = 1
EXECUTE = 2                   # can insert, but won't report insert id
INSERT = 3
RECURSE = 4                     # recursive generator

def db_result(*val, **named):
    assert len(val) == 0 or len(named) == 0
    if len(val) == 1:
        return (DONE, val[0])
    elif len(val) > 1:
        return (DONE, val)
    elif len(named) != 0:
        t = namedtuple('_', named.keys())
        return (DONE, t(**named))
    else:
        return (DONE, None)

def db_input(t, sql, *args):
    return (t, (sql, args))

db_query = partial(db_input, QUERY)
db_execute = partial(db_input, EXECUTE)
db_insert = partial(db_input, INSERT)

def db_recurse(g):
    return (RECURSE, g)

def for_recurse(f):
    def wrapper(*args, **kwargs):
        return db_recurse(f(*args, **kwargs))
    return wrapper

def _exec(cur, g):
    what, val = g.next()
    while True:
        if what == DONE:
            return val
        elif what == RECURSE:
            try:
                r = _exec(cur, val)
            except StopIteration:
                r = None
            except Exception as e:
                what, val = g.throw(type(e), e)
                continue
            what, val = g.send(r)<commit_msg>close the generator after it yields back the result, it won't be run anyway<commit_after>from collections import namedtuple
from functools import partial


DONE = 0
QUERY = 1
EXECUTE = 2                   # can insert, but won't report insert id
INSERT = 3
RECURSE = 4                     # recursive generator

def db_result(*val, **named):
    assert len(val) == 0 or len(named) == 0
    if len(val) == 1:
        return (DONE, val[0])
    elif len(val) > 1:
        return (DONE, val)
    elif len(named) != 0:
        t = namedtuple('_', named.keys())
        return (DONE, t(**named))
    else:
        return (DONE, None)

def db_input(t, sql, *args):
    return (t, (sql, args))

db_query = partial(db_input, QUERY)
db_execute = partial(db_input, EXECUTE)
db_insert = partial(db_input, INSERT)

def db_recurse(g):
    return (RECURSE, g)

def for_recurse(f):
    def wrapper(*args, **kwargs):
        return db_recurse(f(*args, **kwargs))
    return wrapper

def _exec(cur, g):
    what, val = g.next()
    while True:
        if what == DONE:
            g.close()
            return val
        elif what == RECURSE:
            try:
                r = _exec(cur, val)
            except StopIteration:
                r = None
            except Exception as e:
                what, val = g.throw(type(e), e)
                continue
            what, val = g.send(r)
{'fraction_non_alphanumeric': 0.11705310396409872, 'fraction_numerical': 0.008227374719521317, 'mean_word_length': 1.9921700223713648, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2159450', 'n_tokens_mistral': 903, 'n_tokens_neox': 854, 'n_words': 271}