<commit_before># Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import re
import time
from uuid import uuid4


FAST_FORWARD_ONLY = 1
MERGE_ALWAYS = 2
MERGE_IF_NECESSARY = 3
CHERRY_PICK = 4


class Pipeline(object):
    """A top-level pipeline such as check, gate, post, etc."""
    def __init__(self, name):
        self.name = name
        self.job_trees = {}  # project -> JobTree
        self.manager = None
        self.queues = []

    def __repr__(self):
        return '<Pipeline %s>' % self.name

    def setManager(self, manager):
        self.manager = manager

    def addProject(self, project):
        job_tree = JobTree(None)  # Null job == job tree root
        self.job_trees[project] = job_tree
        return job_tree

    def getProjects(self):
        return self.job_trees.keys()

    def addQueue(self, queue):
        self.queues.append(queue)

    def getQueue(self, project):
        for queue in self.queues:
            if project in queue.projects:
                return queue
        return None

    def getJobTree(self, project):
        tree = self.job_trees.get(project)
        return tree

    def getJobs(self, changeish):
        tree = self.getJobTree(changeish.project)
        if not tree:
            return []
        return changeish.filterJobs(tree.getJobs())

    def _findJobsToRun(self, job_trees, changeish):
        torun = []
        if changeish.change_ahead:
            # Only run jobs if any 'hold' jobs on the change ahead
            # have completed successfully.
            if self.isHoldingFollowingChanges(changeish.change_ahead):
                return []
        for tree in job_trees:
            job = tree.job
            result = None
            if job:
                if not job.changeMatches(changeish):
                    continue
                build = changeish.current_build_set.getBuild(job.name)
                if build:
                    result = build.result
                else:
                    # There is no build for the root of this job tree,
                    # so we should run it.
                    torun.append(job)
            # If there is no job, this is a null job tree, and we should
            # run all of its jobs.
            if result == 'SUCCESS' or not job:
                torun.extend(self._findJobsToRun(tree.job_trees, changeish))
        return torun

    def findJobsToRun(self, changeish):
        tree = self.getJobTree(changeish.project)
        if not tree:
            return []
        return self._findJobsToRun(tree.job_trees, changeish)

    def areAllJobsComplete(self, changeish):
        for job in self.getJobs(changeish):
            build = changeish.current_build_set.getBuild(job.name)
            if not build or not build.result:
                return False
        return True

    def didAllJobsSucceed(self, changeish):
        for job in self.getJobs(changeish):
            if not job.voting:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if not build:
                return False
            if build.result != 'SUCCESS':
                return False
        return True

    def didAnyJobFail(self, changeish):
        for job in self.getJobs(changeish):
            if not job.voting:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if build and build.result == 'FAILURE':
                return True
        return False

    def isHoldingFollowingChanges(self, changeish):
        for job in self.getJobs(changeish):
            if not job.hold_following_changes:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if not build:
                return True
            if build.result != 'SUCCESS':
                return True
        if not changeish.change_ahead:
            return False
        return self.isHoldingFollowingChanges(changeish.change_ahead)

    def setResult(self, changeish, build):
        if build.result != 'SUCCESS':
            # Get a JobTree from a Job so we can find only its dependent jobs
            root = self.getJobTree(changeish.project)
            tree = root.getJobTreeForJob(build.job)
            for job in tree.getJobs():
                fakebuild = Build(job, None)
                fakebuild.result = 'SKIPPED'
                changeish.addBuild(fakebuild)

    def setUnableToMerge(self, changeish):
        changeish.current_build_set.unable_to_merge = True
        root = self.getJobTree(changeish.project)
        for job in root.getJobs():
            fakebuild = Build(job, None)
            fakebuild.result = 'SKIPPED'
            changeish.addBuild(fakebuild)

    def setDequeuedNeedingChange(self, changeish):
        changeish.dequeued_needing_change = True
        root = self.getJobTree(changeish.project)
        for job in root.getJobs():
            fakebuild = Build(job, None)
            fakebuild.result = 'SKIPPED'
            changeish.addBuild(fakebuild)

    def getChangesInQueue(self):
        changes = []
        for shared_queue in self.queues:
            changes.extend(shared_queue.queue)
        return changes

    def getAllChanges(self):
        changes = []
        for shared_queue in self.queues:
            changes.extend(shared_queue.queue)
            changes.extend(shared_queue.severed_heads)
        return changes

    def formatStatusHTML(self):
        ret = ''
        for queue in self.queues:
            if len(self.queues) > 1:
                s = 'Change queue: %s' % queue.name
                ret += s + '\n'
                ret += '-' * len(s) + '\n'
            for head in queue.getHeads():
                ret += self.formatStatus(head, html=True)
        return ret

    def formatStatus(self, changeish, indent=0, html=False):
        indent_str = ' ' * indent
        ret = ''
        if html and hasattr(changeish, 'url') and changeish.url is not None:
            ret += '%sProject %s change <a href="%s">%s</a>\n' % (
                indent_str,
                changeish.project.name,
                changeish.url,
                changeish._id())
        else:
            ret += '%sProject %s change %s\n' % (indent_str,
                                                 changeish.project.name,
                                                 changeish._id())
        for job in self.getJobs(changeish):
            build = changeish.current_build_set.getBuild(job.name)
            if build:
                result = build.result
            else:
                result = None
            job_name = job.name
            if not job.voting:
                voting = ' (non-voting)'
            else:
                voting = ''
            if html:
                if build:
                    url = build.url
                else:
                    url = None
                if url is not None:
                    job_name = '<a href="%s">%s</a>' % (url, job_name)
            ret += '%s  %s: %s%s' % (indent_str, job_name, result, voting)
            ret += '\n'
        if changeish.change_behind:
            ret += '%sFollowed by:\n' % (indent_str)
            ret += self.formatStatus(changeish.change_behind, indent + 2, html)
        return ret


class ChangeQueue(object):
    """DependentPipelines have multiple parallel queues shared by
    different projects; this is one of them.  For instance, there may
    a queue shared by interrelated projects foo and bar, and a second
    queue for independent project baz.  Pipelines have one or more
    PipelineQueues."""
    def __init__(self, pipeline, dependent=True):
        self.pipeline = pipeline
        self.name = ''
        self.projects = []
        self._jobs = set()
        self.queue = []
        self.severed_heads = []
        self.dependent = dependent

    def __repr__(self):
        return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)

    def getJobs(self):
        return self._jobs

    def addProject(self, project):
        if project not in self.projects:
            self.projects.append(project)
            names = [x.name for x in self.projects]
            names.sort()
            self.name = ', '.join(names)
            self._jobs |= set(self.pipeline.getJobTree(project).getJobs())

    def enqueueChange(self, change):
        if self.queue:
            change.change_ahead = self.queue[-1]
            change.change_ahead.change_behind = change
        self.queue.append(change)

    def dequeueChange(self, change):
        if change in self.queue:
            self.queue.remove(change)
        if change in self.severed_heads:
            self.severed_heads.remove(change)
        if change.change_ahead:
            change.change_ahead.change_behind = change.change_behind
        if change.change_behind:
            change.change_behind.change_ahead = change.change_ahead
        change.change_ahead = None
        change.change_behind = None

    def addSeveredHead(self, change):
        self.severed_heads.append(change)
<commit_msg>Don't set ahead/behind on independent queues.<commit_after># Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import re
import time
from uuid import uuid4


FAST_FORWARD_ONLY = 1
MERGE_ALWAYS = 2
MERGE_IF_NECESSARY = 3
CHERRY_PICK = 4


class Pipeline(object):
    """A top-level pipeline such as check, gate, post, etc."""
    def __init__(self, name):
        self.name = name
        self.job_trees = {}  # project -> JobTree
        self.manager = None
        self.queues = []

    def __repr__(self):
        return '<Pipeline %s>' % self.name

    def setManager(self, manager):
        self.manager = manager

    def addProject(self, project):
        job_tree = JobTree(None)  # Null job == job tree root
        self.job_trees[project] = job_tree
        return job_tree

    def getProjects(self):
        return self.job_trees.keys()

    def addQueue(self, queue):
        self.queues.append(queue)

    def getQueue(self, project):
        for queue in self.queues:
            if project in queue.projects:
                return queue
        return None

    def getJobTree(self, project):
        tree = self.job_trees.get(project)
        return tree

    def getJobs(self, changeish):
        tree = self.getJobTree(changeish.project)
        if not tree:
            return []
        return changeish.filterJobs(tree.getJobs())

    def _findJobsToRun(self, job_trees, changeish):
        torun = []
        if changeish.change_ahead:
            # Only run jobs if any 'hold' jobs on the change ahead
            # have completed successfully.
            if self.isHoldingFollowingChanges(changeish.change_ahead):
                return []
        for tree in job_trees:
            job = tree.job
            result = None
            if job:
                if not job.changeMatches(changeish):
                    continue
                build = changeish.current_build_set.getBuild(job.name)
                if build:
                    result = build.result
                else:
                    # There is no build for the root of this job tree,
                    # so we should run it.
                    torun.append(job)
            # If there is no job, this is a null job tree, and we should
            # run all of its jobs.
            if result == 'SUCCESS' or not job:
                torun.extend(self._findJobsToRun(tree.job_trees, changeish))
        return torun

    def findJobsToRun(self, changeish):
        tree = self.getJobTree(changeish.project)
        if not tree:
            return []
        return self._findJobsToRun(tree.job_trees, changeish)

    def areAllJobsComplete(self, changeish):
        for job in self.getJobs(changeish):
            build = changeish.current_build_set.getBuild(job.name)
            if not build or not build.result:
                return False
        return True

    def didAllJobsSucceed(self, changeish):
        for job in self.getJobs(changeish):
            if not job.voting:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if not build:
                return False
            if build.result != 'SUCCESS':
                return False
        return True

    def didAnyJobFail(self, changeish):
        for job in self.getJobs(changeish):
            if not job.voting:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if build and build.result == 'FAILURE':
                return True
        return False

    def isHoldingFollowingChanges(self, changeish):
        for job in self.getJobs(changeish):
            if not job.hold_following_changes:
                continue
            build = changeish.current_build_set.getBuild(job.name)
            if not build:
                return True
            if build.result != 'SUCCESS':
                return True
        if not changeish.change_ahead:
            return False
        return self.isHoldingFollowingChanges(changeish.change_ahead)

    def setResult(self, changeish, build):
        if build.result != 'SUCCESS':
            # Get a JobTree from a Job so we can find only its dependent jobs
            root = self.getJobTree(changeish.project)
            tree = root.getJobTreeForJob(build.job)
            for job in tree.getJobs():
                fakebuild = Build(job, None)
                fakebuild.result = 'SKIPPED'
                changeish.addBuild(fakebuild)

    def setUnableToMerge(self, changeish):
        changeish.current_build_set.unable_to_merge = True
        root = self.getJobTree(changeish.project)
        for job in root.getJobs():
            fakebuild = Build(job, None)
            fakebuild.result = 'SKIPPED'
            changeish.addBuild(fakebuild)

    def setDequeuedNeedingChange(self, changeish):
        changeish.dequeued_needing_change = True
        root = self.getJobTree(changeish.project)
        for job in root.getJobs():
            fakebuild = Build(job, None)
            fakebuild.result = 'SKIPPED'
            changeish.addBuild(fakebuild)

    def getChangesInQueue(self):
        changes = []
        for shared_queue in self.queues:
            changes.extend(shared_queue.queue)
        return changes

    def getAllChanges(self):
        changes = []
        for shared_queue in self.queues:
            changes.extend(shared_queue.queue)
            changes.extend(shared_queue.severed_heads)
        return changes

    def formatStatusHTML(self):
        ret = ''
        for queue in self.queues:
            if len(self.queues) > 1:
                s = 'Change queue: %s' % queue.name
                ret += s + '\n'
                ret += '-' * len(s) + '\n'
            for head in queue.getHeads():
                ret += self.formatStatus(head, html=True)
        return ret

    def formatStatus(self, changeish, indent=0, html=False):
        indent_str = ' ' * indent
        ret = ''
        if html and hasattr(changeish, 'url') and changeish.url is not None:
            ret += '%sProject %s change <a href="%s">%s</a>\n' % (
                indent_str,
                changeish.project.name,
                changeish.url,
                changeish._id())
        else:
            ret += '%sProject %s change %s\n' % (indent_str,
                                                 changeish.project.name,
                                                 changeish._id())
        for job in self.getJobs(changeish):
            build = changeish.current_build_set.getBuild(job.name)
            if build:
                result = build.result
            else:
                result = None
            job_name = job.name
            if not job.voting:
                voting = ' (non-voting)'
            else:
                voting = ''
            if html:
                if build:
                    url = build.url
                else:
                    url = None
                if url is not None:
                    job_name = '<a href="%s">%s</a>' % (url, job_name)
            ret += '%s  %s: %s%s' % (indent_str, job_name, result, voting)
            ret += '\n'
        if changeish.change_behind:
            ret += '%sFollowed by:\n' % (indent_str)
            ret += self.formatStatus(changeish.change_behind, indent + 2, html)
        return ret


class ChangeQueue(object):
    """DependentPipelines have multiple parallel queues shared by
    different projects; this is one of them.  For instance, there may
    a queue shared by interrelated projects foo and bar, and a second
    queue for independent project baz.  Pipelines have one or more
    PipelineQueues."""
    def __init__(self, pipeline, dependent=True):
        self.pipeline = pipeline
        self.name = ''
        self.projects = []
        self._jobs = set()
        self.queue = []
        self.severed_heads = []
        self.dependent = dependent

    def __repr__(self):
        return '<ChangeQueue %s: %s>' % (self.pipeline.name, self.name)

    def getJobs(self):
        return self._jobs

    def addProject(self, project):
        if project not in self.projects:
            self.projects.append(project)
            names = [x.name for x in self.projects]
            names.sort()
            self.name = ', '.join(names)
            self._jobs |= set(self.pipeline.getJobTree(project).getJobs())

    def enqueueChange(self, change):
        if self.dependent and self.queue:
            change.change_ahead = self.queue[-1]
            change.change_ahead.change_behind = change
        self.queue.append(change)

    def dequeueChange(self, change):
        if change in self.queue:
            self.queue.remove(change)
        if change in self.severed_heads:
            self.severed_heads.remove(change)
        if change.change_ahead:
            change.change_ahead.change_behind = change.change_behind
        if change.change_behind:
            change.change_behind.change_ahead = change.change_ahead
        change.change_ahead = None
        change.change_behind = None

    def addSeveredHead(self, change):
        self.severed_heads.append(change)

{'fraction_non_alphanumeric': 0.08579254741847124, 'fraction_numerical': 0.0017571967543542303, 'mean_word_length': 1.8023171614771905, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169662', 'n_tokens_mistral': 5454, 'n_tokens_neox': 5203, 'n_words': 1481}