<commit_before>from __future__ import absolute_import, unicode_literals

import logging
import os

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'promgen.settings')
logger = logging.getLogger(__name__)

app = Celery('promgen')

# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()


@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))


# Because of the way that celery is de-coupled, it can be quite difficult to
# get an instance of 'self' when converting a class method into a celery task.
# as a way to get around this, we can mimick a @task(bind=True) call so that we
# get a self instance to the Celery task, and then we can set a __klass__
# attribute that we can use to get to our other class functions
def wrap_send(cls):
    if hasattr(cls, '_send'):
        logger.debug('Wrapping %s', cls)
        cls._send = app.task(cls._send, bind=True, lazy=False)
        cls._send.__klass__ = cls()
    return cls<commit_msg>Switch to using a per-worker queue<commit_after>from __future__ import absolute_import, unicode_literals

import logging
import os
import platform

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'promgen.settings')
logger = logging.getLogger(__name__)

app = Celery('promgen')

# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()

app.conf.task_default_queue = platform.node()

@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))


# Because of the way that celery is de-coupled, it can be quite difficult to
# get an instance of 'self' when converting a class method into a celery task.
# as a way to get around this, we can mimick a @task(bind=True) call so that we
# get a self instance to the Celery task, and then we can set a __klass__
# attribute that we can use to get to our other class functions
def wrap_send(cls):
    if hasattr(cls, '_send'):
        logger.debug('Wrapping %s', cls)
        cls._send = app.task(cls._send, bind=True, lazy=False)
        cls._send.__klass__ = cls()
    return cls
{'fraction_non_alphanumeric': 0.08994708994708994, 'fraction_numerical': 0.0007054673721340388, 'mean_word_length': 4.549902152641879, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '303434', 'n_tokens_mistral': 854, 'n_tokens_neox': 844, 'n_words': 330}