<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import logging

from django.conf import settings
from polyaxon_schemas.utils import TaskType
from polyaxon_spawner.spawner import K8SExperimentSpawner

from api.settings import CeleryTasks
from api.celery_api import app as celery_app
from experiments.models import ExperimentJob
from experiments.task_status import (
    RedisExperimentStatus,
)

logger = logging.getLogger('polyaxon.api.experiments')


@celery_app.task(name=CeleryTasks.EXPERIMENTS_START)
def start_experiment(experiment_id):
    from experiments.models import Experiment, ExperimentStatus, ExperimentLifeCycle

    try:
        experiment = Experiment.objects.get(id=experiment_id)
    except Experiment.DoesNotExist:
        logger.info('Experiment id `{}` does not exist'.format(experiment_id))
        return

    # Update experiment status to show that its started
    ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.SCHEDULED)

    # Use spawner to start the experiment
    spawner = K8SExperimentSpawner(project_id=experiment.project.id,
                                   spec_id=experiment.spec.id if experiment.spec else '',
                                   experiment_id=experiment.id,
                                   specification=experiment.config,
                                   k8s_config=settings.K8S_CONFIG)
    resp = spawner.start_experiment()

    # Get the number of jobs this experiment started
    ExperimentJob.objects.create(experiment=experiment, definition=resp[TaskType.MASTER])
    for worker in resp[TaskType.WORKER]:
        ExperimentJob.objects.create(experiment=experiment, definition=worker)
        for ps in resp[TaskType.PS]:
            ExperimentJob.objects.create(experiment=experiment, definition=ps)

    # Add the experiment to the list of experiments to monitor
    RedisExperimentStatus.monitor(experiment_id)

<commit_msg>Update experiments tasks<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import logging

from django.conf import settings
from polyaxon_schemas.utils import TaskType
from polyaxon_spawner.spawner import K8SExperimentSpawner

from api.celery_api import app as celery_app
from api.settings import CeleryTasks
from experiments.models import ExperimentJob
from libs.redis_db import RedisExperimentStatus

logger = logging.getLogger('polyaxon.tasks.experiments')


@celery_app.task(name=CeleryTasks.EXPERIMENTS_START)
def start_experiment(experiment_id):
    from experiments.models import Experiment, ExperimentStatus, ExperimentLifeCycle

    try:
        experiment = Experiment.objects.get(id=experiment_id)
    except Experiment.DoesNotExist:
        logger.info('Experiment id `{}` does not exist'.format(experiment_id))
        return

    # Update experiment status to show that its started
    ExperimentStatus.objects.create(experiment=experiment, status=ExperimentLifeCycle.SCHEDULED)

    # Use spawner to start the experiment
    spawner = K8SExperimentSpawner(project_id=experiment.project.id,
                                   spec_id=experiment.spec.id if experiment.spec else '',
                                   experiment_id=experiment.id,
                                   specification=experiment.config,
                                   k8s_config=settings.K8S_CONFIG)
    resp = spawner.start_experiment()

    # Get the number of jobs this experiment started
    ExperimentJob.objects.create(experiment=experiment, definition=resp[TaskType.MASTER])
    for worker in resp[TaskType.WORKER]:
        ExperimentJob.objects.create(experiment=experiment, definition=worker)
        for ps in resp[TaskType.PS]:
            ExperimentJob.objects.create(experiment=experiment, definition=ps)

    # Add the experiment to the list of experiments to monitor
    RedisExperimentStatus.monitor(experiment_id)

{'fraction_non_alphanumeric': 0.06888720666161999, 'fraction_numerical': 0.0025233409033560434, 'mean_word_length': 3.5045454545454544, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '627631', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1067, 'n_words': 248}