<commit_before>from datetime import timedelta
from django.utils.timezone import now as tz_now

from awx.main.scheduler.partial import (
    JobDict,
    ProjectUpdateDict,
    InventoryUpdateDict,
    SystemJobDict,
    AdHocCommandDict,
    WorkflowJobDict,
)


class DependencyGraph(object):
    PROJECT_UPDATES = 'project_updates'
    INVENTORY_UPDATES = 'inventory_updates'

    JOB_TEMPLATE_JOBS = 'job_template_jobs'
    JOB_PROJECT_IDS = 'job_project_ids'
    JOB_INVENTORY_IDS = 'job_inventory_ids'

    SYSTEM_JOB = 'system_job'
    INVENTORY_SOURCE_UPDATES = 'inventory_source_updates'
    WORKFLOW_JOB_TEMPLATES_JOBS = 'workflow_job_template_jobs'

    LATEST_PROJECT_UPDATES = 'latest_project_updates'
    LATEST_INVENTORY_UPDATES = 'latest_inventory_updates'

    INVENTORY_SOURCES = 'inventory_source_ids'

    def __init__(self, *args, **kwargs):
        self.data = {}
        # project_id -> True / False
        self.data[self.PROJECT_UPDATES] = {}
        # inventory_id -> True / False
        self.data[self.INVENTORY_UPDATES] = {}
        # job_template_id -> True / False
        self.data[self.JOB_TEMPLATE_JOBS] = {}

        '''
        Track runnable job related project and inventory to ensure updates
        don't run while a job needing those resources is running.
        '''
        # project_id -> True / False
        self.data[self.JOB_PROJECT_IDS] = {}
        # inventory_id -> True / False
        self.data[self.JOB_INVENTORY_IDS] = {}

        # inventory_source_id -> True / False
        self.data[self.INVENTORY_SOURCE_UPDATES] = {}
        # True / False
        self.data[self.SYSTEM_JOB] = True
        # workflow_job_template_id -> True / False
        self.data[self.WORKFLOW_JOB_TEMPLATES_JOBS] = {}

        # project_id -> latest ProjectUpdateLatestDict
        self.data[self.LATEST_PROJECT_UPDATES] = {}
        # inventory_source_id -> latest InventoryUpdateLatestDict
        self.data[self.LATEST_INVENTORY_UPDATES] = {}

        # inventory_id -> [inventory_source_ids]
        self.data[self.INVENTORY_SOURCES] = {}

    def add_latest_project_update(self, job):
        self.data[self.LATEST_PROJECT_UPDATES][job['project_id']] = job

    def add_latest_inventory_update(self, job):
        self.data[self.LATEST_INVENTORY_UPDATES][job['inventory_source_id']] = job

    def add_inventory_sources(self, inventory_id, inventory_sources):
        self.data[self.INVENTORY_SOURCES][inventory_id] = inventory_sources

    def get_inventory_sources(self, inventory_id):
        return self.data[self.INVENTORY_SOURCES].get(inventory_id, [])

    def get_now(self):
        return tz_now()

    '''
    JobDict
    
    Presume that job is related to a project that is update on launch
    '''
    def should_update_related_project(self, job):
        now = self.get_now()
        latest_project_update = self.data[self.LATEST_PROJECT_UPDATES].get(job['project_id'], None)
        if not latest_project_update:
            return True

        # TODO: Other finished, failed cases? i.e. error ?
        if latest_project_update['status'] in ['failed', 'canceled']:
            return True

        '''
        This is a bit of fuzzy logic.
        If the latest project update has a created time == job_created_time-1 
        then consider the project update found. This is so we don't enter an infinite loop
        of updating the project when cache timeout is 0.
        '''
        if latest_project_update['project__scm_update_cache_timeout'] == 0 and \
                latest_project_update['launch_type'] == 'dependency' and \
                latest_project_update['created'] == job['created'] - timedelta(seconds=1):
            return False
<commit_msg>like inventory updates, check if project update deps already processed<commit_after>from datetime import timedelta
from django.utils.timezone import now as tz_now

from awx.main.scheduler.partial import (
    JobDict,
    ProjectUpdateDict,
    InventoryUpdateDict,
    SystemJobDict,
    AdHocCommandDict,
    WorkflowJobDict,
)


class DependencyGraph(object):
    PROJECT_UPDATES = 'project_updates'
    INVENTORY_UPDATES = 'inventory_updates'

    JOB_TEMPLATE_JOBS = 'job_template_jobs'
    JOB_PROJECT_IDS = 'job_project_ids'
    JOB_INVENTORY_IDS = 'job_inventory_ids'

    SYSTEM_JOB = 'system_job'
    INVENTORY_SOURCE_UPDATES = 'inventory_source_updates'
    WORKFLOW_JOB_TEMPLATES_JOBS = 'workflow_job_template_jobs'

    LATEST_PROJECT_UPDATES = 'latest_project_updates'
    LATEST_INVENTORY_UPDATES = 'latest_inventory_updates'

    INVENTORY_SOURCES = 'inventory_source_ids'

    def __init__(self, *args, **kwargs):
        self.data = {}
        # project_id -> True / False
        self.data[self.PROJECT_UPDATES] = {}
        # inventory_id -> True / False
        self.data[self.INVENTORY_UPDATES] = {}
        # job_template_id -> True / False
        self.data[self.JOB_TEMPLATE_JOBS] = {}

        '''
        Track runnable job related project and inventory to ensure updates
        don't run while a job needing those resources is running.
        '''
        # project_id -> True / False
        self.data[self.JOB_PROJECT_IDS] = {}
        # inventory_id -> True / False
        self.data[self.JOB_INVENTORY_IDS] = {}

        # inventory_source_id -> True / False
        self.data[self.INVENTORY_SOURCE_UPDATES] = {}
        # True / False
        self.data[self.SYSTEM_JOB] = True
        # workflow_job_template_id -> True / False
        self.data[self.WORKFLOW_JOB_TEMPLATES_JOBS] = {}

        # project_id -> latest ProjectUpdateLatestDict
        self.data[self.LATEST_PROJECT_UPDATES] = {}
        # inventory_source_id -> latest InventoryUpdateLatestDict
        self.data[self.LATEST_INVENTORY_UPDATES] = {}

        # inventory_id -> [inventory_source_ids]
        self.data[self.INVENTORY_SOURCES] = {}

    def add_latest_project_update(self, job):
        self.data[self.LATEST_PROJECT_UPDATES][job['project_id']] = job

    def add_latest_inventory_update(self, job):
        self.data[self.LATEST_INVENTORY_UPDATES][job['inventory_source_id']] = job

    def add_inventory_sources(self, inventory_id, inventory_sources):
        self.data[self.INVENTORY_SOURCES][inventory_id] = inventory_sources

    def get_inventory_sources(self, inventory_id):
        return self.data[self.INVENTORY_SOURCES].get(inventory_id, [])

    def get_now(self):
        return tz_now()

    '''
    JobDict
    
    Presume that job is related to a project that is update on launch
    '''
    def should_update_related_project(self, job):
        now = self.get_now()

        # Already processed dependencies for this job
        if job.data['dependent_jobs__id'] is not None:
            return False

        latest_project_update = self.data[self.LATEST_PROJECT_UPDATES].get(job['project_id'], None)
        if not latest_project_update:
            return True

        # TODO: Other finished, failed cases? i.e. error ?
        if latest_project_update['status'] in ['failed', 'canceled']:
            return True

        '''
        This is a bit of fuzzy logic.
        If the latest project update has a created time == job_created_time-1 
        then consider the project update found. This is so we don't enter an infinite loop
        of updating the project when cache timeout is 0.
        '''
        if latest_project_update['project__scm_update_cache_timeout'] == 0 and \
                latest_project_update['launch_type'] == 'dependency' and \
                latest_project_update['created'] == job['created'] - timedelta(seconds=1):
            return False

{'fraction_non_alphanumeric': 0.09020584764651894, 'fraction_numerical': 0.0010489052051920807, 'mean_word_length': 2.9299330242143227, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808340', 'n_tokens_mistral': 2504, 'n_tokens_neox': 2367, 'n_words': 482}