<commit_before>from __future__ import absolute_import, print_function
import concurrent.futures
import logging
import multiprocessing

import s.cached


_size = multiprocessing.cpu_count() + 2


@s.cached.func
def _pool():
    logging.debug('new process pool, size: %s', _size)
    return concurrent.futures.ProcessPoolExecutor(_size)


def new(fn, *a, **kw):
    daemon = kw.pop('_daemon', True)
    obj = multiprocessing.Process(target=fn, args=a, kwargs=kw)
    obj.daemon = daemon
    obj.start()
    return obj


def wait(*fns):
    objs = [new(fn) for fn in fns]
    [obj.join() for obj in objs]


def submit(fn, *a, **kw):
    return _pool().submit(fn, *a, **kw)
<commit_msg>hacking<commit_after>from __future__ import absolute_import, print_function
import concurrent.futures
import logging
import multiprocessing

import s.cached


_size = multiprocessing.cpu_count() + 2


@s.cached.func
def _pool():
    logging.debug('new process pool, size: %s', _size)
    return concurrent.futures.ProcessPoolExecutor(_size)


def new(fn, *a, **kw):
    daemon = kw.pop('_daemon', True)
    obj = multiprocessing.Process(target=fn, args=a, kwargs=kw)
    obj.daemon = daemon
    obj.start()
    return obj


def wait(*fns):
    objs = [new(fn) for fn in fns]
    [obj.join() for obj in objs]


def submit(fn, *a, **kw):
    return _pool().submit(fn, *a, **kw)


# TODO monitor, like wait, but blows up when anybody dies

{'fraction_non_alphanumeric': 0.1304654442877292, 'fraction_numerical': 0.0014104372355430183, 'mean_word_length': 3.9964788732394365, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3042380', 'n_tokens_mistral': 517, 'n_tokens_neox': 500, 'n_words': 138}