<commit_before>from datetime import timedelta

from kombu import Exchange, Queue

from polyaxon.config_manager import config

CELERY_TRACK_STARTED = True

AMQP_URL = config.get_string('POLYAXON_AMQP_URL')
RABBITMQ_USER = config.get_string('POLYAXON_RABBITMQ_USER', is_optional=True)
RABBITMQ_PASSWORD = config.get_string('POLYAXON_RABBITMQ_PASSWORD',
                                      is_secret=True,
                                      is_optional=True)
BROKER_POOL_LIMIT = None

if RABBITMQ_USER and RABBITMQ_PASSWORD:
    CELERY_BROKER_URL = 'amqp://{user}:{password}@{url}'.format(
        user=RABBITMQ_USER,
        password=<PASSWORD>,
        url=AMQP_URL
    )

# todo: fix me at some point
CELERY_BROKER_URL = 'amqp://{url}'.format(url=AMQP_URL)

INTERNAL_EXCHANGE = config.get_string('POLYAXON_INTERNAL_EXCHANGE',
                                      is_optional=True,
                                      default='internal')

CELERY_RESULT_BACKEND = config.get_string('POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL')
CELERYD_PREFETCH_MULTIPLIER = config.get_int('POLYAXON_CELERYD_PREFETCH_MULTIPLIER')

CELERY_TASK_ALWAYS_EAGER = config.get_boolean('POLYAXON_CELERY_ALWAYS_EAGER')
if CELERY_TASK_ALWAYS_EAGER:
    BROKER_TRANSPORT = 'memory'

CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_IGNORE_RESULT = True
CELERY_IGNORE_RESULT = True
CELERY_HARD_TIME_LIMIT_DELAY = config.get_int('POLYAXON_CELERY_HARD_TIME_LIMIT_DELAY',
                                              is_optional=True,
                                              default=180)
HEALTH_CHECK_WORKER_TIMEOUT = config.get_int('POLYAXON_HEALTH_CHECK_WORKER_TIMEOUT',
                                             is_optional=True,
                                             default=4)
GROUP_CHUNKS = config.get_int('POLYAXON_GROUP_CHUNKS',
                              is_optional=True,
                              default=50)


class Intervals(object):
    """All intervals are in seconds"""
    OPERATIONS_DEFAULT_RETRY_DELAY = config.get_int(
        'POLYAXON_INTERVALS_OPERATIONS_DEFAULT_RETRY_DELAY',
        is_optional=True,
        default=60)
    OPERATIONS_MAX_RETRY_DELAY = config.get_int(
        'POLYAXON_INTERVALS_OPERATIONS_MAX_RETRY_DELAY',
        is_optional=True,
        default=60 * 60)
    PIPELINES_SCHEDULER = config.get_int(
        'POLYAXON_INTERVALS_PIPELINES_SCHEDULER',
        is_optional=True,
        default=30)
    EXPERIMENTS_SCHEDULER = config.get_int(
        'POLYAXON_INTERVALS_EXPERIMENTS_SCHEDULER',
        is_optional=True,
        default=30)
    EXPERIMENTS_SYNC = config.get_int(
        'POLYAXON_INTERVALS_EXPERIMENTS_SYNC',<commit_msg>Add max tasks per child config<commit_after>from datetime import timedelta

from kombu import Exchange, Queue

from polyaxon.config_manager import config

CELERY_TRACK_STARTED = True

AMQP_URL = config.get_string('POLYAXON_AMQP_URL')
RABBITMQ_USER = config.get_string('POLYAXON_RABBITMQ_USER', is_optional=True)
RABBITMQ_PASSWORD = config.get_string('POLYAXON_RABBITMQ_PASSWORD',
                                      is_secret=True,
                                      is_optional=True)
BROKER_POOL_LIMIT = None

if RABBITMQ_USER and RABBITMQ_PASSWORD:
    CELERY_BROKER_URL = 'amqp://{user}:{password}@{url}'.format(
        user=RABBITMQ_USER,
        password=<PASSWORD>,
        url=AMQP_URL
    )

# todo: fix me at some point
CELERY_BROKER_URL = 'amqp://{url}'.format(url=AMQP_URL)

INTERNAL_EXCHANGE = config.get_string('POLYAXON_INTERNAL_EXCHANGE',
                                      is_optional=True,
                                      default='internal')

CELERY_RESULT_BACKEND = config.get_string('POLYAXON_REDIS_CELERY_RESULT_BACKEND_URL')
CELERYD_PREFETCH_MULTIPLIER = config.get_int('POLYAXON_CELERYD_PREFETCH_MULTIPLIER')

CELERY_TASK_ALWAYS_EAGER = config.get_boolean('POLYAXON_CELERY_ALWAYS_EAGER')
if CELERY_TASK_ALWAYS_EAGER:
    BROKER_TRANSPORT = 'memory'

CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_IGNORE_RESULT = True
CELERY_IGNORE_RESULT = True
CELERY_HARD_TIME_LIMIT_DELAY = config.get_int('POLYAXON_CELERY_HARD_TIME_LIMIT_DELAY',
                                              is_optional=True,
                                              default=180)
HEALTH_CHECK_WORKER_TIMEOUT = config.get_int('POLYAXON_HEALTH_CHECK_WORKER_TIMEOUT',
                                             is_optional=True,
                                             default=4)
CELERYD_MAX_TASKS_PER_CHILD = config.get_int('POLYAXON_CELERYD_MAX_TASKS_PER_CHILD',
                                             is_optional=True,
                                             default=100)
GROUP_CHUNKS = config.get_int('POLYAXON_GROUP_CHUNKS',
                              is_optional=True,
                              default=50)


class Intervals(object):
    """All intervals are in seconds"""
    OPERATIONS_DEFAULT_RETRY_DELAY = config.get_int(
        'POLYAXON_INTERVALS_OPERATIONS_DEFAULT_RETRY_DELAY',
        is_optional=True,
        default=60)
    OPERATIONS_MAX_RETRY_DELAY = config.get_int(
        'POLYAXON_INTERVALS_OPERATIONS_MAX_RETRY_DELAY',
        is_optional=True,
        default=60 * 60)
    PIPELINES_SCHEDULER = config.get_int(
        'POLYAXON_INTERVALS_PIPELINES_SCHEDULER',
        is_optional=True,
        default=30)
    EXPERIMENTS_SCHEDULER = config.get_int(
        'POLYAXON_INTERVALS_EXPERIMENTS_SCHEDULER',
        is_optional=True,
        default=30)
    EXPERIMENTS_SYNC = config.get_int(
        'POLYAXON_INTERVALS_EXPERIMENTS_SYNC',
{'fraction_non_alphanumeric': 0.07306414962419157, 'fraction_numerical': 0.0061178115714036006, 'mean_word_length': 2.6632522407170294, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1785816', 'n_tokens_mistral': 2048, 'n_tokens_neox': 1963, 'n_words': 228}