<commit_before>from oebakery import die, err, warn, info, debug
from oelite import *
import sys, os, copy

class OEliteRunQueue:


    def __init__(self, db, cookbook, config, rebuild=None, relax=None,
                 depth_first=True):
        self.db = db
        self.cookbook = cookbook
        self.config = config
        # 1: --rebuild, 2: --rebuildall, 3: --reallyrebuildall
        self.rebuild = rebuild
        self.relax = relax
        self.depth_first = depth_first
        self._assume_provided = (self.config.getVar("ASSUME_PROVIDED", 1)
                                or "").split()
        self.runable = []
        self.metahashable = []
        return


    def assume_provided(self, item):
        if isinstance(item, str) or isinstance(item, unicode):
            return item in self._assume_provided
        return self.db.get_item(item) in self._assume_provided


    def add_something(self, something, task_name):
        task_name = self.db.task_name_id(task_name)
        return (self.add_rprovider(something, task_name) or
                self.add_provider(something, task_name) or
                self.add_recipe(something, task_name) or
                self.add_package(something, task_name))


    def add_provider(self, item, task_name):
        package = self.get_provider(item, allow_no_provider=True)
        if not package:
            return False
        return self._add_package(package, task_name)


    def add_rprovider(self, ritem, task_name):
        package = self.get_rprovider(ritem, allow_no_provider=True)
        if not package:
            return False
        return self._add_package(package, task_name)


    def add_recipe(self, name, task_name):
        recipe = self.db.get_recipe_id(name=name)
        if not recipe:
            return False
        return self._add_recipe(recipe, task_name, primary=True)


    def add_package(self, name, task_name):
        package = self.db.get_package_id(name=name)
        return self._add_package(package, task_name)


    def _add_package(self, package_id, task_name):
        if not package_id:
            return False
        return self._add_recipe(
            self.db.get_recipe_id(package=package_id), task_name, primary=True)


    def _add_recipe(self, recipe_id, task_name, primary=False):
        if primary:
            primary_recipe_id = recipe_id
        else:
            primary_recipe_id = None
        primary_task = self.db.get_task_id(recipe_id, task_name)
        if not primary_task:
            raise NoSuchTask("%s:%s"%(
                    self.db.get_recipe_name(recipe_id),
                    self.db.get_task(task_name)))

        alltasks = set()
        addedtasks = set([primary_task])

        while addedtasks:
            self.db.add_runq_tasks(addedtasks)

            alltasks.update(addedtasks)
            newtasks = set()
            for task in addedtasks:

                recipe_id = self.db.recipe_id({"task": task})
                recipe_data = self.cookbook[recipe_id].data

                if recipe_id == primary_recipe_id:
                    is_primary_recipe = True
                else:
                    is_primary_recipe = False

                # set rebuild flag (based on
                # --rebuild/--rebuildall/--reallyrebuildall)
                if ((self.rebuild >= 1 and is_primary_recipe) or
                    (self.rebuild == 2 and
                     recipe_data.getVar("REBUILDALL_SKIP") != "1") or
                    (self.rebuild == 3)):
                    self.db.set_runq_task_build(task)

                # set relax flag (based on --sloppy/--relaxed)
                if ((self.relax == 2 and not is_primary_recipe) or
                    (self.relax == 1 and
                     (not is_primary_recipe and
                      recipe_data.getVar("RELAXED", True)))):
                    self.db.set_runq_task_relax(task)

                try:
                    # task_dependencies should return tuple:
                    # task_depends: list of task_id's
                    # package_depends: list of (task_id, package_id)
                    # package_rdepends: list of (task_id, package_id)
                    (task_depends, package_depends, package_rdepends) = \
                        self.task_dependencies(task)
                    self.db.add_runq_task_depends(task, task_depends)
                    self.db.add_runq_package_depends(task, package_depends)
                    self.db.add_runq_package_rdepends(task, package_rdepends)
                    newtasks.update(task_depends)
                    newtasks.update([d[0] for d in package_depends])
                    newtasks.update([d[0] for d in package_rdepends])
                except RecursiveDepends, e:
                    recipe = self.db.get_recipe_name({"task":task})
                    task = self.db.get_task(task=task)
                    raise RecursiveDepends(e.args[0], "%s:%s"%(recipe, task))
<commit_msg>lib/oelite: fix rebuildall bug<commit_after>from oebakery import die, err, warn, info, debug
from oelite import *
import sys, os, copy

class OEliteRunQueue:


    def __init__(self, db, cookbook, config, rebuild=None, relax=None,
                 depth_first=True):
        self.db = db
        self.cookbook = cookbook
        self.config = config
        # 1: --rebuild, 2: --rebuildall, 3: --reallyrebuildall
        self.rebuild = rebuild
        self.relax = relax
        self.depth_first = depth_first
        self._assume_provided = (self.config.getVar("ASSUME_PROVIDED", 1)
                                or "").split()
        self.runable = []
        self.metahashable = []
        return


    def assume_provided(self, item):
        if isinstance(item, str) or isinstance(item, unicode):
            return item in self._assume_provided
        return self.db.get_item(item) in self._assume_provided


    def add_something(self, something, task_name):
        task_name = self.db.task_name_id(task_name)
        return (self.add_rprovider(something, task_name) or
                self.add_provider(something, task_name) or
                self.add_recipe(something, task_name) or
                self.add_package(something, task_name))


    def add_provider(self, item, task_name):
        package = self.get_provider(item, allow_no_provider=True)
        if not package:
            return False
        return self._add_package(package, task_name)


    def add_rprovider(self, ritem, task_name):
        package = self.get_rprovider(ritem, allow_no_provider=True)
        if not package:
            return False
        return self._add_package(package, task_name)


    def add_recipe(self, name, task_name):
        recipe = self.db.get_recipe_id(name=name)
        if not recipe:
            return False
        return self._add_recipe(recipe, task_name, primary=True)


    def add_package(self, name, task_name):
        package = self.db.get_package_id(name=name)
        return self._add_package(package, task_name)


    def _add_package(self, package_id, task_name):
        if not package_id:
            return False
        return self._add_recipe(
            self.db.get_recipe_id(package=package_id), task_name, primary=True)


    def _add_recipe(self, recipe_id, task_name, primary=False):
        if primary:
            primary_recipe_id = recipe_id
        else:
            primary_recipe_id = None
        primary_task = self.db.get_task_id(recipe_id, task_name)
        if not primary_task:
            raise NoSuchTask("%s:%s"%(
                    self.db.get_recipe_name(recipe_id),
                    self.db.get_task(task_name)))

        alltasks = set()
        addedtasks = set([primary_task])

        while addedtasks:
            self.db.add_runq_tasks(addedtasks)

            alltasks.update(addedtasks)
            newtasks = set()
            for task in addedtasks:

                recipe_id = self.db.recipe_id({"task": task})
                recipe_data = self.cookbook[recipe_id].data

                if recipe_id == primary_recipe_id:
                    is_primary_recipe = True
                else:
                    is_primary_recipe = False

                # set rebuild flag (based on
                # --rebuild/--rebuildall/--reallyrebuildall)
                if ((self.rebuild >= 1 and is_primary_recipe) or
                    (self.rebuild == 2 and
                     recipe_data.getVar("REBUILDALL_SKIP", True) != "1") or
                    (self.rebuild == 3)):
                    self.db.set_runq_task_build(task)

                # set relax flag (based on --sloppy/--relaxed)
                if ((self.relax == 2 and not is_primary_recipe) or
                    (self.relax == 1 and
                     (not is_primary_recipe and
                      recipe_data.getVar("RELAXED", True)))):
                    self.db.set_runq_task_relax(task)

                try:
                    # task_dependencies should return tuple:
                    # task_depends: list of task_id's
                    # package_depends: list of (task_id, package_id)
                    # package_rdepends: list of (task_id, package_id)
                    (task_depends, package_depends, package_rdepends) = \
                        self.task_dependencies(task)
                    self.db.add_runq_task_depends(task, task_depends)
                    self.db.add_runq_package_depends(task, package_depends)
                    self.db.add_runq_package_rdepends(task, package_rdepends)
                    newtasks.update(task_depends)
                    newtasks.update([d[0] for d in package_depends])
                    newtasks.update([d[0] for d in package_rdepends])
                except RecursiveDepends, e:
                    recipe = self.db.get_recipe_name({"task":task})
                    task = self.db.get_task(task=task)
                    raise RecursiveDepends(e.args[0], "%s:%s"%(recipe, task))

{'fraction_non_alphanumeric': 0.08886885081655145, 'fraction_numerical': 0.002604949403867348, 'mean_word_length': 1.8357954545454545, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1156882', 'n_tokens_mistral': 3055, 'n_tokens_neox': 2873, 'n_words': 598}