<commit_before>import os
import os.path
import copy
import string
import logging
from random import choice
import re
import time
import socket

import json
import paramiko
import tarfile
import shutil
from django.conf import settings

from nailgun.models import Cluster, Node, Role, Recipe
from nailgun.helpers import SshConnect, DeployGenerator
from nailgun.task_helpers import task_with_callbacks, TaskPool, topol_sort
from nailgun.exceptions import SSHError, EmptyListError, DeployError
from nailgun.provision import ProvisionConfig
from nailgun.provision import ProvisionFactory
from nailgun.provision.model.profile import Profile as ProvisionProfile
from nailgun.provision.model.node import Node as ProvisionNode
from nailgun.provision.model.power import Power as ProvisionPower

from celery import current_app
from celery.utils import LOG_LEVELS
from celery.log import Logging


current_app.conf.CELERYD_LOG_LEVEL = LOG_LEVELS[settings.CELERYLOGLEVEL]
celery_logging = Logging(current_app)
celery_logging.setup_logger(logfile=settings.CELERYLOGFILE)
logger = celery_logging.get_default_logger()


def resolve_recipe_deps(graph, recipe):
    if recipe.recipe not in graph:
        graph[recipe.recipe] = []
    for r in recipe.depends.all():
        graph[recipe.recipe].append(r.recipe)
        resolve_recipe_deps(graph, r)


@task_with_callbacks
def update_cluster_status(*args):
    # FIXME(mihgen):
    # We have to do this ugly trick because chord precedes first argument
    if isinstance(args[0], list):
        args = args[1:]
    cluster_id = args[0]

    return cluster_id


def node_set_error_status(node_id):
    node = Node.objects.get(id=node_id)
    node.status = "error"
    node.save()


@task_with_callbacks(name='Deploy Cluster')
def deploy_cluster(cluster_id):

    graph = {}
    for recipe in Recipe.objects.filter(
                recipe__in=DeployGenerator.recipes(cluster_id)):
        resolve_recipe_deps(graph, recipe)
        #graph[recipe.recipe] = [r.recipe for r in recipe.depends.all()]

    # NOTE(mihgen): Installation components dependency resolution
    # From nodes.roles.recipes we know recipes that needs to be applied
    # We have to apply them in an order according to specified dependencies
    # To sort in an order, we use DFS(Depth First Traversal) over DAG graph
    sorted_recipes = topol_sort(graph)
    tree = TaskPool()
    # first element in sorted_recipes is the first recipe we have to apply
    installed = []
    for r in sorted_recipes:
        recipe = Recipe.objects.get(recipe=r)
        # We need to find nodes with these recipes
        roles = recipe.roles.all()<commit_msg>Reverted creation of graph of recipes to simpler one.<commit_after>import os
import os.path
import copy
import string
import logging
from random import choice
import re
import time
import socket

import json
import paramiko
import tarfile
import shutil
from django.conf import settings

from nailgun.models import Cluster, Node, Role, Recipe
from nailgun.helpers import SshConnect, DeployGenerator
from nailgun.task_helpers import task_with_callbacks, TaskPool, topol_sort
from nailgun.exceptions import SSHError, EmptyListError, DeployError
from nailgun.provision import ProvisionConfig
from nailgun.provision import ProvisionFactory
from nailgun.provision.model.profile import Profile as ProvisionProfile
from nailgun.provision.model.node import Node as ProvisionNode
from nailgun.provision.model.power import Power as ProvisionPower

from celery import current_app
from celery.utils import LOG_LEVELS
from celery.log import Logging


current_app.conf.CELERYD_LOG_LEVEL = LOG_LEVELS[settings.CELERYLOGLEVEL]
celery_logging = Logging(current_app)
celery_logging.setup_logger(logfile=settings.CELERYLOGFILE)
logger = celery_logging.get_default_logger()


@task_with_callbacks
def update_cluster_status(*args):
    # FIXME(mihgen):
    # We have to do this ugly trick because chord precedes first argument
    if isinstance(args[0], list):
        args = args[1:]
    cluster_id = args[0]

    return cluster_id


def node_set_error_status(node_id):
    node = Node.objects.get(id=node_id)
    node.status = "error"
    node.save()


@task_with_callbacks(name='Deploy Cluster')
def deploy_cluster(cluster_id):

    graph = {}
    for recipe in Recipe.objects.filter(
                recipe__in=DeployGenerator.recipes(cluster_id)):
        graph[recipe.recipe] = [r.recipe for r in recipe.depends.all()]

    # NOTE(mihgen): Installation components dependency resolution
    # From nodes.roles.recipes we know recipes that needs to be applied
    # We have to apply them in an order according to specified dependencies
    # To sort in an order, we use DFS(Depth First Traversal) over DAG graph
    sorted_recipes = topol_sort(graph)
    tree = TaskPool()
    # first element in sorted_recipes is the first recipe we have to apply
    installed = []
    for r in sorted_recipes:
        recipe = Recipe.objects.get(recipe=r)
        # We need to find nodes with these recipes
        roles = recipe.roles.all()
{'fraction_non_alphanumeric': 0.06730007917656373, 'fraction_numerical': 0.0011876484560570072, 'mean_word_length': 4.347089947089947, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1746434', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1490, 'n_words': 469}