<commit_before># -*- coding: utf-8 -*-
#
# NOTE: this snippet should come *after* the other celery_*
#       because it uses the BROKER_URL that should have been
#       defined in these.
#

#from datetime import timedelta
import djcelery
djcelery.setup_loader()
from celery.schedules import crontab
from kombu import Exchange, Queue

# Avoid sharing the same celery states file
# when multiple workers run on the same machine.
try:
    index = sys.argv.index('--hostname')
except:
    CELERYD_STATE_DB = 'celery.states'
else:
    # get 'medium' from 'medium.worker-03.1flow.io'
    CELERYD_STATE_DB = 'celery.states.{0}'.format(
                            sys.argv[index + 1].split('.', 1)[0])
    del index

CELERYD_PREFETCH_MULTIPLIER = 1

CELERY_DEFAULT_QUEUE = 'medium'

CELERY_QUEUES = (
    Queue('high', Exchange('high'), routing_key='high'),
    Queue('medium', Exchange('medium'), routing_key='medium'),
    Queue('low', Exchange('low'), routing_key='low'),
    Queue('fetch', Exchange('fetch'), routing_key='fetch'),
    Queue('swarm', Exchange('swarm'), routing_key='swarm'),
    Queue('background', Exchange('background'), routing_key='background'),
)

BROKER_URL = os.environ.get('BROKER_URL')

CELERY_RESULT_BACKEND = BROKER_URL
CELERY_RESULT_PERSISTENT = True

# Allow to recover from any unknown crash.
CELERY_ACKS_LATE = True

# Sometimes, Ask asks us to enable this to debug issues.
# BTW, it will save some CPU cycles.
CELERY_DISABLE_RATE_LIMITS = True

# Allow our remote workers to get tasks faster if they have a
# slow internet connection (yes Gurney, I'm thinking of you).
CELERY_MESSAGE_COMPRESSION = 'gzip'

# Avoid long running and retried tasks to be run over-and-over again.
BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 86400}

# Half a day is enough
CELERY_TASK_RESULT_EXPIRES = 43200

# The default beiing 5000, we need more than this.<commit_msg>Disable broker heartbeat, this just breaks things in flower.<commit_after># -*- coding: utf-8 -*-
#
# NOTE: this snippet should come *after* the other celery_*
#       because it uses the BROKER_URL that should have been
#       defined in these.
#

#from datetime import timedelta
import djcelery
djcelery.setup_loader()
from celery.schedules import crontab
from kombu import Exchange, Queue

# Avoid sharing the same celery states file
# when multiple workers run on the same machine.
try:
    index = sys.argv.index('--hostname')
except:
    CELERYD_STATE_DB = 'celery.states'
else:
    # get 'medium' from 'medium.worker-03.1flow.io'
    CELERYD_STATE_DB = 'celery.states.{0}'.format(
                            sys.argv[index + 1].split('.', 1)[0])
    del index

CELERYD_PREFETCH_MULTIPLIER = 1

CELERY_DEFAULT_QUEUE = 'medium'

CELERY_QUEUES = (
    Queue('high', Exchange('high'), routing_key='high'),
    Queue('medium', Exchange('medium'), routing_key='medium'),
    Queue('low', Exchange('low'), routing_key='low'),
    Queue('fetch', Exchange('fetch'), routing_key='fetch'),
    Queue('swarm', Exchange('swarm'), routing_key='swarm'),
    Queue('background', Exchange('background'), routing_key='background'),
)

BROKER_URL = os.environ.get('BROKER_URL')

# Disabling the heartbeat because workers seems often disabled in flower,
# thanks to http://stackoverflow.com/a/14831904/654755
BROKER_HEARTBEAT=0

CELERY_RESULT_BACKEND = BROKER_URL
CELERY_RESULT_PERSISTENT = True

# Allow to recover from any unknown crash.
CELERY_ACKS_LATE = True

# Sometimes, Ask asks us to enable this to debug issues.
# BTW, it will save some CPU cycles.
CELERY_DISABLE_RATE_LIMITS = True

# Allow our remote workers to get tasks faster if they have a
# slow internet connection (yes Gurney, I'm thinking of you).
CELERY_MESSAGE_COMPRESSION = 'gzip'

# Avoid long running and retried tasks to be run over-and-over again.
BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 86400}

# Half a day is enough
CELERY_TASK_RESULT_EXPIRES = 43200

# The default beiing 5000, we need more than this.
{'fraction_non_alphanumeric': 0.11119515885022693, 'fraction_numerical': 0.015380736258194654, 'mean_word_length': 4.441700960219479, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5198389', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1272, 'n_words': 387}