<commit_before>from experiment_groups.utils import get_valid_experiment_group
from polyaxon.celery_api import app as celery_app
from polyaxon.settings import HPCeleryTasks, Intervals
from runner.hp_search import base


def create(experiment_group):
    experiment_group.iteration_manager.create_iteration()
    experiments = base.create_group_experiments(experiment_group=experiment_group)
    experiment_group.iteration_manager.add_iteration_experiments(
        experiment_ids=[xp.id for xp in experiments])
    hp_hyperband_start.apply_async((experiment_group.id,), countdown=1)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_CREATE)
def hp_hyperband_create(experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_START, bind=True, max_retries=None)
def hp_hyperband_start(self, experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)

    hp_hyperband_iterate.delay(experiment_group_id=experiment_group_id)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_ITERATE)
def hp_hyperband_iterate(experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(iteration=iteration_config.iteration,
                                        bracket_iteration=iteration_config.bracket_iteration):
        hp_hyperband_create.delay(experiment_group_id=experiment_group_id)
        return

    if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
                                            bracket_iteration=iteration_config.bracket_iteration):
        iteration_manager.reduce_configs(experiment_group=experiment_group)
        hp_hyperband_start.delay(experiment_group_id=experiment_group_id)
<commit_msg>Update hp hyperband reduce config<commit_after>from experiment_groups.utils import get_valid_experiment_group
from polyaxon.celery_api import app as celery_app
from polyaxon.settings import HPCeleryTasks, Intervals
from runner.hp_search import base


def create(experiment_group):
    experiment_group.iteration_manager.create_iteration()
    experiments = base.create_group_experiments(experiment_group=experiment_group)
    experiment_group.iteration_manager.add_iteration_experiments(
        experiment_ids=[xp.id for xp in experiments])
    hp_hyperband_start.apply_async((experiment_group.id,), countdown=1)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_CREATE)
def hp_hyperband_create(experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    create(experiment_group)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_START, bind=True, max_retries=None)
def hp_hyperband_start(self, experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    should_retry = base.start_group_experiments(experiment_group=experiment_group)
    if should_retry:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)

    hp_hyperband_iterate.delay(experiment_group_id=experiment_group_id)


@celery_app.task(name=HPCeleryTasks.HP_HYPERBAND_ITERATE)
def hp_hyperband_iterate(experiment_group_id):
    experiment_group = get_valid_experiment_group(experiment_group_id=experiment_group_id)
    if not experiment_group:
        return

    iteration_config = experiment_group.iteration_config
    iteration_manager = experiment_group.iteration_manager
    search_manager = experiment_group.search_manager

    iteration_manager.update_iteration()

    if search_manager.should_reschedule(iteration=iteration_config.iteration,
                                        bracket_iteration=iteration_config.bracket_iteration):
        hp_hyperband_create.delay(experiment_group_id=experiment_group_id)
        return

    if search_manager.should_reduce_configs(iteration=iteration_config.iteration,
                                            bracket_iteration=iteration_config.bracket_iteration):
        iteration_manager.reduce_configs()
        hp_hyperband_start.delay(experiment_group_id=experiment_group_id)

{'fraction_non_alphanumeric': 0.05728848114169215, 'fraction_numerical': 0.00040774719673802244, 'mean_word_length': 5.217997465145754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6800452', 'n_tokens_mistral': 1523, 'n_tokens_neox': 1500, 'n_words': 182}