<commit_before>"""
Celery configuration file

To setup celery properly we need the broker and backend urls, which
can be set by environment variable MEERKAT_BROKER_URL.

In this config we set up a beat schedule with a timedelta in seconds
given by the config.interval value.

"""
from datetime import timedelta, datetime
from celery.schedules import crontab
import logging
import os
import yaml

from meerkat_abacus.config import config

BROKER_URL = 'amqp://guest@rabbit//'
CELERY_RESULT_BACKEND = 'rpc://guest@rabbit//'

new_url = os.environ.get("MEERKAT_BROKER_URL")
if new_url:
    BROKER_URL = new_url
    CELERY_RESULT_BACKEND = new_url

CELERY_TASK_SERIALIZER = 'yaml'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json', 'yaml']
CELERY_ENABLE_UTC = True
CELERYD_MAX_TASKS_PER_CHILD = 1  # To help with memory constraints


CELERYBEAT_SCHEDULE = {}
CELERYBEAT_SCHEDULE['cleanup_downloads'] = {
    'task': 'meerkat_abacus.tasks.cleanup_downloads',
    'schedule': crontab(minute=16, hour='*')
}

if config.consul_enabled:
    CELERYBEAT_SCHEDULE['consul'] = {
        'task': 'meerkat_abacus.tasks.consul_export',
        'schedule': 90.0,
        'args': (yaml.dump(config), )
    }


# Each report will need it's own sending schedule.
# Add them from the country config to the celery schedule here.
# Only add if the mailing root is set - empty env variable silences reports.
if config.mailing_root:

    schedule = config.country_config['reports_schedule']

    for report in schedule:

        # Set up some stuff.<commit_msg>Default consul export internal 24h.<commit_after>"""
Celery configuration file

To setup celery properly we need the broker and backend urls, which
can be set by environment variable MEERKAT_BROKER_URL.

In this config we set up a beat schedule with a timedelta in seconds
given by the config.interval value.

"""
from datetime import timedelta, datetime
from celery.schedules import crontab
import logging
import os
import yaml

from meerkat_abacus.config import config

BROKER_URL = 'amqp://guest@rabbit//'
CELERY_RESULT_BACKEND = 'rpc://guest@rabbit//'

new_url = os.environ.get("MEERKAT_BROKER_URL")
if new_url:
    BROKER_URL = new_url
    CELERY_RESULT_BACKEND = new_url

CELERY_TASK_SERIALIZER = 'yaml'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json', 'yaml']
CELERY_ENABLE_UTC = True
CELERYD_MAX_TASKS_PER_CHILD = 1  # To help with memory constraints


CELERYBEAT_SCHEDULE = {}
CELERYBEAT_SCHEDULE['cleanup_downloads'] = {
    'task': 'meerkat_abacus.tasks.cleanup_downloads',
    'schedule': crontab(minute=16, hour='*')
}

if config.consul_enabled:
    CELERYBEAT_SCHEDULE['consul'] = {
        'task': 'meerkat_abacus.tasks.consul_export',
        'schedule': 24*3600, #daily
        'args': (yaml.dump(config), )
    }


# Each report will need it's own sending schedule.
# Add them from the country config to the celery schedule here.
# Only add if the mailing root is set - empty env variable silences reports.
if config.mailing_root:

    schedule = config.country_config['reports_schedule']

    for report in schedule:

        # Set up some stuff.
{'fraction_non_alphanumeric': 0.08998085513720484, 'fraction_numerical': 0.005424377791959157, 'mean_word_length': 4.461672473867596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1184939', 'n_tokens_mistral': 1106, 'n_tokens_neox': 1058, 'n_words': 302}