<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import logging

from random import shuffle

from docker.errors import DockerException
from polyaxon_schemas.utils import SEARCH_METHODS

from polyaxon.settings import CeleryTasks, Intervals
from polyaxon.celery_api import app as celery_app
from experiments.tasks import build_experiment
from projects.models import ExperimentGroup, Project
from repos import dockerize
from repos.dockerize import get_job_image_info
from repos.models import Repo
from spawner import scheduler

logger = logging.getLogger('polyaxon.tasks.projects')


@celery_app.task(name=CeleryTasks.EXPERIMENTS_START_GROUP, bind=True, max_retries=None)
def start_group_experiments(self, experiment_group_id):
    try:
        experiment_group = ExperimentGroup.objects.get(id=experiment_group_id)
    except ExperimentGroup.DoesNotExist:
        logger.info('ExperimentGroup `{}` was not found.'.format(experiment_group_id))
        if self.request.retries < 2:
            logger.info('Trying again for ExperimentGroup `{}`.'.format(experiment_group_id))
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)

        logger.info('Something went wrong, '
                    'the ExperimentGroup `{}` does not exist anymore.'.format(experiment_group_id))
        return

    pending_experiments = list(experiment_group.pending_experiments)
    experiment_to_start = experiment_group.n_experiments_to_start

    if experiment_group.search_method == SEARCH_METHODS.RANDOM:
        shuffle(pending_experiments)

    while experiment_to_start > 0 and pending_experiments:
        experiment = pending_experiments.pop()
        build_experiment.delay(experiment_id=experiment.id)
        experiment_to_start -= 1

    if pending_experiments:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)


def get_valid_project(project_id):
    try:
        return Project.objects.get(id=project_id)
    except Project.DoesNotExist:
        logger.info('Project id `{}` does not exist'.format(project_id))
        return None


@celery_app.task(name=CeleryTasks.PROJECTS_TENSORBOARD_START, ignore_result=True)
def start_tensorboard(project_id):
    project = get_valid_project(project_id)
    if not project or not project.tensorboard:
        return None
    scheduler.start_tensorboard(project)


@celery_app.task(name=CeleryTasks.PROJECTS_TENSORBOARD_STOP, ignore_result=True)
def stop_tensorboard(project_id):
    project = get_valid_project(project_id)
    if not project:
        return None
    scheduler.stop_tensorboard(project)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_BUILD, ignore_result=True)
def build_notebook(project_id):
    project = get_valid_project(project_id)
    if not project or not project.notebook:
        return None

    # docker image
    try:
        status = dockerize.build_notebook_job(project=project, job=project.notebook)
    except DockerException:
        return
    except Repo.DoesNotExist:
        logger.warning('No code was found for this project')
        return

    if not status:
        return

    # Now we can start the notebook
    start_notebook.delay(project_id=project_id)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_START, ignore_result=True)
def start_notebook(project_id):
    project = get_valid_project(project_id)
    if not project or not project.notebook:
        return None

    try:
        image_name, image_tag = get_job_image_info(project=project, job=project.notebook)
    except ValueError as e:
        logger.warning('Could not start the notebook, %s', e)
        return
    job_docker_image = '{}:{}'.format(image_name, image_tag)
    logger.info('Start notebook with built image `{}`'.format(job_docker_image))

    scheduler.start_notebook(project, image=job_docker_image)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_STOP, ignore_result=True)
def stop_notebook(project_id):
    project = get_valid_project(project_id)
    if not project:
        return None

    scheduler.stop_notebook(project)
<commit_msg>Check if project has plugin jobs before start new ones<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import logging

from random import shuffle

from docker.errors import DockerException
from polyaxon_schemas.utils import SEARCH_METHODS

from polyaxon.settings import CeleryTasks, Intervals
from polyaxon.celery_api import app as celery_app
from experiments.tasks import build_experiment
from projects.models import ExperimentGroup, Project
from repos import dockerize
from repos.dockerize import get_job_image_info
from repos.models import Repo
from spawner import scheduler

logger = logging.getLogger('polyaxon.tasks.projects')


@celery_app.task(name=CeleryTasks.EXPERIMENTS_START_GROUP, bind=True, max_retries=None)
def start_group_experiments(self, experiment_group_id):
    try:
        experiment_group = ExperimentGroup.objects.get(id=experiment_group_id)
    except ExperimentGroup.DoesNotExist:
        logger.info('ExperimentGroup `{}` was not found.'.format(experiment_group_id))
        if self.request.retries < 2:
            logger.info('Trying again for ExperimentGroup `{}`.'.format(experiment_group_id))
            self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)

        logger.info('Something went wrong, '
                    'the ExperimentGroup `{}` does not exist anymore.'.format(experiment_group_id))
        return

    pending_experiments = list(experiment_group.pending_experiments)
    experiment_to_start = experiment_group.n_experiments_to_start

    if experiment_group.search_method == SEARCH_METHODS.RANDOM:
        shuffle(pending_experiments)

    while experiment_to_start > 0 and pending_experiments:
        experiment = pending_experiments.pop()
        build_experiment.delay(experiment_id=experiment.id)
        experiment_to_start -= 1

    if pending_experiments:
        # Schedule another task
        self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER)


def get_valid_project(project_id):
    try:
        return Project.objects.get(id=project_id)
    except Project.DoesNotExist:
        logger.info('Project id `{}` does not exist'.format(project_id))
        return None


@celery_app.task(name=CeleryTasks.PROJECTS_TENSORBOARD_START, ignore_result=True)
def start_tensorboard(project_id):
    project = get_valid_project(project_id)
    if not project or not project.tensorboard or project.has_tensorboard:
        return None
    scheduler.start_tensorboard(project)


@celery_app.task(name=CeleryTasks.PROJECTS_TENSORBOARD_STOP, ignore_result=True)
def stop_tensorboard(project_id):
    project = get_valid_project(project_id)
    if not project:
        return None
    scheduler.stop_tensorboard(project)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_BUILD, ignore_result=True)
def build_notebook(project_id):
    project = get_valid_project(project_id)
    if not project or not project.notebook:
        return None

    # docker image
    try:
        status = dockerize.build_notebook_job(project=project, job=project.notebook)
    except DockerException:
        return
    except Repo.DoesNotExist:
        logger.warning('No code was found for this project')
        return

    if not status:
        return

    # Now we can start the notebook
    start_notebook.delay(project_id=project_id)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_START, ignore_result=True)
def start_notebook(project_id):
    project = get_valid_project(project_id)
    if not project or not project.notebook or project.has_notebook:
        return None

    try:
        image_name, image_tag = get_job_image_info(project=project, job=project.notebook)
    except ValueError as e:
        logger.warning('Could not start the notebook, %s', e)
        return
    job_docker_image = '{}:{}'.format(image_name, image_tag)
    logger.info('Start notebook with built image `{}`'.format(job_docker_image))

    scheduler.start_notebook(project, image=job_docker_image)


@celery_app.task(name=CeleryTasks.PROJECTS_NOTEBOOK_STOP, ignore_result=True)
def stop_notebook(project_id):
    project = get_valid_project(project_id)
    if not project:
        return None

    scheduler.stop_notebook(project)

{'fraction_non_alphanumeric': 0.07686781609195402, 'fraction_numerical': 0.0009578544061302681, 'mean_word_length': 4.26007556675063, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5255956', 'n_tokens_mistral': 2604, 'n_tokens_neox': 2513, 'n_words': 529}