<commit_before>__author__ = '<NAME>'

import collections
from datetime import datetime
from threading import RLock

from synergy.db.dao.job_dao import JobDao
from synergy.db.model.job import Job
from synergy.conf import context
from synergy.conf import settings
from synergy.system import time_helper
from synergy.system.time_qualifier import *
from synergy.system.decorator import thread_safe
from synergy.scheduler.scheduler_constants import COLLECTION_JOB_HOURLY, COLLECTION_JOB_DAILY, \
    COLLECTION_JOB_MONTHLY, COLLECTION_JOB_YEARLY
from synergy.scheduler.tree import MultiLevelTree
from synergy.scheduler.state_machine_continuous import StateMachineContinuous
from synergy.scheduler.state_machine_dicrete import StateMachineDiscrete
from synergy.scheduler.state_machine_simple_dicrete import StateMachineSimpleDiscrete
from synergy.scheduler.state_machine_freerun import StateMachineFreerun


class Timetable(object):
    """ Timetable holds all known process trees, where every node presents a timeperiod-driven job"""

    def __init__(self, logger):
        self.lock = RLock()
        self.logger = logger
        self.job_dao = JobDao(self.logger)

        # state_machines must be constructed before the trees
        self.state_machines = self._construct_state_machines()

        # self.trees contain all of the trees and manages much of their life cycle
        # remember to enlist here all trees the system is working with
        self.trees = self._construct_trees_from_context()

        self._register_dependencies()
        self.load_tree()
        self.build_trees()
        self.validate()

    def _construct_state_machines(self):
        """ :return: dict in format <state_machine_common_name: instance_of_the_state_machine> """
        state_machines = dict()
        for state_machine in [StateMachineContinuous(self.logger, self),
                              StateMachineDiscrete(self.logger, self),
                              StateMachineSimpleDiscrete(self.logger, self),
                              StateMachineFreerun(self.logger)]:
            state_machines[state_machine.name] = state_machine
        return state_machines

    def _construct_trees_from_context(self):
        trees = dict()
        for tree_name, context_entry in context.timetable_context.items():
            tree = MultiLevelTree(process_names=context_entry.enclosed_processes,
                                  timetable=self,
                                  tree_name=tree_name,
                                  mx_name=context_entry.mx_name,
                                  mx_page=context_entry.mx_page)
            trees[tree_name] = tree
        return trees

    def _register_dependencies(self):
        """ register dependencies between trees"""
        for tree_name, context_entry in context.timetable_context.items():
            tree = self.trees[tree_name]
            assert isinstance(tree, MultiLevelTree)
            for dependent_on in context_entry.dependent_on:
                dependent_on_tree = self.trees[dependent_on]
                assert isinstance(dependent_on_tree, MultiLevelTree)
                tree.register_dependent_on(dependent_on_tree)

    # *** node manipulation methods ***
    def _find_dependant_trees(self, tree_obj):
        """ returns list of trees that are dependent_on given tree_obj """
        dependant_trees = []
        for tree_name, tree in self.trees.items():
            if tree_obj in tree.dependent_on:
                dependant_trees.append(tree)
        return dependant_trees

    def _find_dependant_tree_nodes(self, node_a):
        dependant_nodes = set()
        for tree_b in self._find_dependant_trees(node_a.tree):
            node_b = node_a.find_counterpart_in(tree_b)
            if node_b is None:
                continue
            dependant_nodes.add(node_b)
        return dependant_nodes

    @thread_safe
    def reprocess_tree_node(self, tree_node, tx_context=None):
        """ method reprocesses the node and all its dependants and parent nodes """
        if not tx_context:
            # create transaction context if one was not provided
            tx_context = collections.defaultdict(dict)

        if tree_node.parent is None:
            # do not process 'root' - the only node that has None as 'parent'
            return tx_context
        if tree_node.timeperiod in tx_context[tree_node.process_name]:
            # the node has already been marked for re-processing
            return tx_context

        if tree_node.job_record.is_embryo:
            # the node does not require re-processing
            pass
        else:
            state_machine = self.state_machines[tree_node.job_record.process_name]
            state_machine.reprocess_job(tree_node.job_record)

        tx_context[tree_node.process_name][tree_node.timeperiod] = tree_node
        self.reprocess_tree_node(tree_node.parent, tx_context)

        dependant_nodes = self._find_dependant_tree_nodes(tree_node)
        for node in dependant_nodes:
            self.reprocess_tree_node(node, tx_context)

        return tx_context

    @thread_safe
    def skip_tree_node(self, tree_node):
        """ method skips given node job-record """
        state_machine = self.state_machines[tree_node.process_name]
        state_machine.skip_job(tree_node.job_record)

    @thread_safe
    def assign_job_record(self, tree_node):
        """ - looks for an existing job record in the DB, and if not found
            - creates a job record in STATE_EMBRYO and bind it to the given tree node """
        try:
            job_record = self.job_dao.get_one(tree_node.process_name, tree_node.timeperiod)
        except LookupError:
            state_machine_name = context.process_context[tree_node.process_name].state_machine_name
            state_machine = self.state_machines[state_machine_name]
            job_record = state_machine.create_job(tree_node.process_name, tree_node.timeperiod)
        tree_node.job_record = job_record

    # *** Tree-manipulation methods ***
    @thread_safe
    def get_tree(self, process_name):<commit_msg>#48 state machines dict reference fix<commit_after>__author__ = '<NAME>'

import collections
from datetime import datetime
from threading import RLock

from synergy.db.dao.job_dao import JobDao
from synergy.db.model.job import Job
from synergy.conf import context
from synergy.conf import settings
from synergy.system import time_helper
from synergy.system.time_qualifier import *
from synergy.system.decorator import thread_safe
from synergy.scheduler.scheduler_constants import COLLECTION_JOB_HOURLY, COLLECTION_JOB_DAILY, \
    COLLECTION_JOB_MONTHLY, COLLECTION_JOB_YEARLY
from synergy.scheduler.tree import MultiLevelTree
from synergy.scheduler.state_machine_continuous import StateMachineContinuous
from synergy.scheduler.state_machine_dicrete import StateMachineDiscrete
from synergy.scheduler.state_machine_simple_dicrete import StateMachineSimpleDiscrete
from synergy.scheduler.state_machine_freerun import StateMachineFreerun


class Timetable(object):
    """ Timetable holds all known process trees, where every node presents a timeperiod-driven job"""

    def __init__(self, logger):
        self.lock = RLock()
        self.logger = logger
        self.job_dao = JobDao(self.logger)

        # state_machines must be constructed before the trees
        self.state_machines = self._construct_state_machines()

        # self.trees contain all of the trees and manages much of their life cycle
        # remember to enlist here all trees the system is working with
        self.trees = self._construct_trees_from_context()

        self._register_dependencies()
        self.load_tree()
        self.build_trees()
        self.validate()

    def _construct_state_machines(self):
        """ :return: dict in format <state_machine_common_name: instance_of_the_state_machine> """
        state_machines = dict()
        for state_machine in [StateMachineContinuous(self.logger, self),
                              StateMachineDiscrete(self.logger, self),
                              StateMachineSimpleDiscrete(self.logger, self),
                              StateMachineFreerun(self.logger)]:
            state_machines[state_machine.name] = state_machine
        return state_machines

    def _construct_trees_from_context(self):
        trees = dict()
        for tree_name, context_entry in context.timetable_context.items():
            tree = MultiLevelTree(process_names=context_entry.enclosed_processes,
                                  timetable=self,
                                  tree_name=tree_name,
                                  mx_name=context_entry.mx_name,
                                  mx_page=context_entry.mx_page)
            trees[tree_name] = tree
        return trees

    def _register_dependencies(self):
        """ register dependencies between trees"""
        for tree_name, context_entry in context.timetable_context.items():
            tree = self.trees[tree_name]
            assert isinstance(tree, MultiLevelTree)
            for dependent_on in context_entry.dependent_on:
                dependent_on_tree = self.trees[dependent_on]
                assert isinstance(dependent_on_tree, MultiLevelTree)
                tree.register_dependent_on(dependent_on_tree)

    # *** node manipulation methods ***
    def _find_dependant_trees(self, tree_obj):
        """ returns list of trees that are dependent_on given tree_obj """
        dependant_trees = []
        for tree_name, tree in self.trees.items():
            if tree_obj in tree.dependent_on:
                dependant_trees.append(tree)
        return dependant_trees

    def _find_dependant_tree_nodes(self, node_a):
        dependant_nodes = set()
        for tree_b in self._find_dependant_trees(node_a.tree):
            node_b = node_a.find_counterpart_in(tree_b)
            if node_b is None:
                continue
            dependant_nodes.add(node_b)
        return dependant_nodes

    @thread_safe
    def reprocess_tree_node(self, tree_node, tx_context=None):
        """ method reprocesses the node and all its dependants and parent nodes """
        if not tx_context:
            # create transaction context if one was not provided
            tx_context = collections.defaultdict(dict)

        if tree_node.parent is None:
            # do not process 'root' - the only node that has None as 'parent'
            return tx_context
        if tree_node.timeperiod in tx_context[tree_node.process_name]:
            # the node has already been marked for re-processing
            return tx_context

        if tree_node.job_record.is_embryo:
            # the node does not require re-processing
            pass
        else:
            state_machine_name = context.process_context[tree_node.process_name].state_machine_name
            state_machine = self.state_machines[state_machine_name]
            state_machine.reprocess_job(tree_node.job_record)

        tx_context[tree_node.process_name][tree_node.timeperiod] = tree_node
        self.reprocess_tree_node(tree_node.parent, tx_context)

        dependant_nodes = self._find_dependant_tree_nodes(tree_node)
        for node in dependant_nodes:
            self.reprocess_tree_node(node, tx_context)

        return tx_context

    @thread_safe
    def skip_tree_node(self, tree_node):
        """ method skips given node job-record """
        state_machine_name = context.process_context[tree_node.process_name].state_machine_name
        state_machine = self.state_machines[state_machine_name]
        state_machine.skip_job(tree_node.job_record)

    @thread_safe
    def assign_job_record(self, tree_node):
        """ - looks for an existing job record in the DB, and if not found
            - creates a job record in STATE_EMBRYO and bind it to the given tree node """
        try:
            job_record = self.job_dao.get_one(tree_node.process_name, tree_node.timeperiod)
        except LookupError:
            state_machine_name = context.process_context[tree_node.process_name].state_machine_name
            state_machine = self.state_machines[state_machine_name]
            job_record = state_machine.create_job(tree_node.process_name, tree_node.timeperiod)
        tree_node.job_record = job_record

    # *** Tree-manipulation methods ***
    @thread_safe
    def get_tree(self, process_name):
{'fraction_non_alphanumeric': 0.06428284451586982, 'fraction_numerical': 0.00016070711128967456, 'mean_word_length': 2.761257177394983, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3721159', 'n_tokens_mistral': 3534, 'n_tokens_neox': 3388, 'n_words': 802}