<commit_before>from config import CELERY_TEST_CONFIG, CELERY_TEST_CONFIG_MEMORY
from worker import CeleryWorkerThread


def setup_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1):
    conf = dict(CELERY_TEST_CONFIG.__dict__.items() + config.__dict__.items())
    conf['CELERYD_CONCURRENCY'] = concurrency
    app.config_from_object(conf)


def start_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1):
    setup_celery_worker(app, config=config, concurrency=concurrency)

    worker = CeleryWorkerThread(app)
    worker.daemon = True
    worker.start()
    worker.ready.wait()
    return worker<commit_msg>stop waiting for worker readiness<commit_after>from config import CELERY_TEST_CONFIG, CELERY_TEST_CONFIG_MEMORY
from worker import CeleryWorkerThread


def setup_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1):
    conf = dict(CELERY_TEST_CONFIG.__dict__.items() + config.__dict__.items())
    conf['CELERYD_CONCURRENCY'] = concurrency
    app.config_from_object(conf)


def start_celery_worker(app, config=CELERY_TEST_CONFIG_MEMORY, concurrency=1):
    setup_celery_worker(app, config=config, concurrency=concurrency)

    worker = CeleryWorkerThread(app)
    worker.daemon = True
    worker.start()
    return worker
{'fraction_non_alphanumeric': 0.084375, 'fraction_numerical': 0.003125, 'mean_word_length': 6.362068965517241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4701369', 'n_tokens_mistral': 424, 'n_tokens_neox': 416, 'n_words': 70}