<commit_before>import oebakery
from oebakery import die, err, warn, info, debug
from oelite import *
import sys, os, glob, shutil, datetime

from db import OEliteDB
from recipe import OEliteRecipe
from runq import OEliteRunQueue
import oelite.data

import bb.parse, bb.utils, bb.build, bb.fetch

BB_ENV_WHITELIST = [
    "PATH",
    "PWD",
    "SHELL",
    "TERM",
]

def add_bake_parser_options(parser):
    parser.add_option("-t", "--task",
                      action="store", type="str", default=None,
                      help="task(s) to do")

    parser.add_option("--rebuild",
                      action="append_const", dest="rebuild", const="1",
                      help="rebuild specified recipes")
    parser.add_option("--rebuildall",
                      action="append_const", dest="rebuild", const="2",
                      help="rebuild specified recipes and all dependencies (except cross and native)")
    parser.add_option("--reallyrebuildall",
                      action="append_const", dest="rebuild", const="3",
                      help="rebuild specified recipes and all dependencies")

    parser.add_option("--relaxed",
                      action="append_const", dest="relax", const="1",
                      help="don't rebuild ${RELAXED} recipes because of metadata changes")
    parser.add_option("--sloppy",
                      action="append_const", dest="relax", const="2",
                      help="don't rebuild dependencies because of metadata changes")

    return


def add_show_parser_options(parser):
    parser.add_option("--nohash",
                      action="store_true",
                      help="don't show variables that will be ignored when computing data hash")

    return


def _parse(f, data, include=False):
    try:
        return bb.parse.handle(f, data, include)
    except (IOError, bb.parse.ParseError) as exc:
        die("unable to parse %s: %s" % (f, exc))


class OEliteBaker:

    def __init__(self, options, args, config):

        self.options = options

        self.config = config.createCopy()
        self.import_env()
        self.config = _parse("conf/bitbake.conf", self.config)

        # Handle any INHERITs and inherit the base class
        inherits  = ["base"] + (config.getVar("INHERIT", 1) or "").split()
        for inherit in inherits:
            self.config = _parse("classes/%s.bbclass"%(inherit),
                                 self.config, 1)

        bb.fetch.fetcher_init(self.config)

        # things (ritem, item, recipe, or package) to do
        if args:
            self.things_todo = args
        elif "BB_DEFAULT_THING" in self.config:
            self.things_todo = self.config.getVar("BB_DEFAULT_THING", 1).split()
        else:
            self.things_todo = [ "base-rootfs" ]

        self.appendlist = {}
        self.db = OEliteDB()
        self.prepare_cookbook()

        return


    def __del__(self):
        return


    def import_env(self):
        whitelist = BB_ENV_WHITELIST
        if "BB_ENV_WHITELIST" in os.environ:
            whitelist += os.environ["BB_ENV_WHITELIST"].split()
        if "BB_ENV_WHITELIST" in self.config:
            whitelist += self.config.getVar("BB_ENV_WHITELIST", True).split()
        debug("whitelist=%s"%(whitelist))
        for var in set(os.environ).difference(whitelist):
            del os.environ[var]
        if oebakery.DEBUG:
            debug("Whitelist filtered shell environment:")
            for var in os.environ:
                debug("> %s=%s"%(var, os.environ[var]))
        for var in whitelist:
            if not var in self.config and var in os.environ:
                self.config[var] = os.environ[var]
                debug("importing %s=%s"%(var, os.environ[var]))
        return


    def prepare_cookbook(self):

        self.cookbook = {}

        # collect all available .bb files
        bbrecipes = self.list_bbrecipes()

        # parse all .bb files
        total = len(bbrecipes)
        parsed = 0
        start = datetime.datetime.now()
        for bbrecipe in bbrecipes:
            if not self.options.quiet:
                progress_info("Parsing recipe files", total, parsed)
            data = self.parse_recipe(bbrecipe)
            for extend in data:
                recipe = OEliteRecipe(bbrecipe, extend, data[extend], self.db)
                self.cookbook[recipe.id] = recipe
            parsed += 1
        if not self.options.quiet:
            progress_info("Parsing recipe files", total, parsed)
        if oebakery.DEBUG:
            timing_info("Parsing", start)

        return


    def show(self):

        if len(self.things_todo) == 0:
            die("you must specify something to show")
        elif len(self.things_todo) > 1:
            die("can only show one thing at a time")

        search = self.things_todo[0].split("_", 1)

        if len(search) == 1:
            search_list = [(search[0], "", None)]
        else:
            search_list = [(search[0], "", search[1])]

        for extend in ("native", "sdk", "sdk-cross", "canadian-cross", "cross"):
            if search[0].endswith("-" + extend):
                search_list.append((search[0][:-(len(extend)+1)],
                                    extend, search_list[0][2]))
                break

        found = []
        debug("looking for %s"%(repr(search_list)))
        for search in search_list:
            recipes = self.db.get_recipe_id(
                name=search[0], extend=search[1], version=search[2],
                multiple=True)
            if recipes:
                debug("new found=%s"%(repr(recipes)))
                debug("%s"%(repr(self.db.get_recipe(recipes[0]))))
                found += recipes
        debug("found %s"%(repr(found)))

        if len(found) == 0:
            die("no recipe found")
        elif len(found) > 1:
            chosen = (found[0], self.db.get_recipe(found[0])[1])
            for other in found[1:]:
                debug("chosen=%s other=%s"%(chosen, other))
                version = self.db.get_recipe(other)[1]
                vercmp = bb.utils.vercmp_part(chosen[1], version)
                if vercmp < 0:
                    chosen = (other, version)
                if vercmp == 0:
                    debug("chosen=%s\nother=%s version=%s"%(chosen, other, version))
                    die("you have to be more precise")
            chosen = chosen[0]
        else:
            chosen = found[0]

        recipe = self.cookbook[chosen]

        oelite.data.dump(d=recipe.data, pretty=True,
                         nohash=(not self.options.nohash))

    def bake(self):

        self.setup_tmpdir()

        # task(s) to do
        if self.options.task:
            tasks_todo = self.options.task
        elif "BB_DEFAULT_TASK" in self.config:
            tasks_todo = self.config.getVar("BB_DEFAULT_TASK", 1)<commit_msg>lib/oelite: fix recipe selection in show command (closes #25)<commit_after>import oebakery
from oebakery import die, err, warn, info, debug
from oelite import *
import sys, os, glob, shutil, datetime

from db import OEliteDB
from recipe import OEliteRecipe
from runq import OEliteRunQueue
import oelite.data

import bb.parse, bb.utils, bb.build, bb.fetch

BB_ENV_WHITELIST = [
    "PATH",
    "PWD",
    "SHELL",
    "TERM",
]

def add_bake_parser_options(parser):
    parser.add_option("-t", "--task",
                      action="store", type="str", default=None,
                      help="task(s) to do")

    parser.add_option("--rebuild",
                      action="append_const", dest="rebuild", const="1",
                      help="rebuild specified recipes")
    parser.add_option("--rebuildall",
                      action="append_const", dest="rebuild", const="2",
                      help="rebuild specified recipes and all dependencies (except cross and native)")
    parser.add_option("--reallyrebuildall",
                      action="append_const", dest="rebuild", const="3",
                      help="rebuild specified recipes and all dependencies")

    parser.add_option("--relaxed",
                      action="append_const", dest="relax", const="1",
                      help="don't rebuild ${RELAXED} recipes because of metadata changes")
    parser.add_option("--sloppy",
                      action="append_const", dest="relax", const="2",
                      help="don't rebuild dependencies because of metadata changes")

    return


def add_show_parser_options(parser):
    parser.add_option("--nohash",
                      action="store_true",
                      help="don't show variables that will be ignored when computing data hash")

    return


def _parse(f, data, include=False):
    try:
        return bb.parse.handle(f, data, include)
    except (IOError, bb.parse.ParseError) as exc:
        die("unable to parse %s: %s" % (f, exc))


class OEliteBaker:

    def __init__(self, options, args, config):

        self.options = options

        self.config = config.createCopy()
        self.import_env()
        self.config = _parse("conf/bitbake.conf", self.config)

        # Handle any INHERITs and inherit the base class
        inherits  = ["base"] + (config.getVar("INHERIT", 1) or "").split()
        for inherit in inherits:
            self.config = _parse("classes/%s.bbclass"%(inherit),
                                 self.config, 1)

        bb.fetch.fetcher_init(self.config)

        # things (ritem, item, recipe, or package) to do
        if args:
            self.things_todo = args
        elif "BB_DEFAULT_THING" in self.config:
            self.things_todo = self.config.getVar("BB_DEFAULT_THING", 1).split()
        else:
            self.things_todo = [ "base-rootfs" ]

        self.appendlist = {}
        self.db = OEliteDB()
        self.prepare_cookbook()

        return


    def __del__(self):
        return


    def import_env(self):
        whitelist = BB_ENV_WHITELIST
        if "BB_ENV_WHITELIST" in os.environ:
            whitelist += os.environ["BB_ENV_WHITELIST"].split()
        if "BB_ENV_WHITELIST" in self.config:
            whitelist += self.config.getVar("BB_ENV_WHITELIST", True).split()
        debug("whitelist=%s"%(whitelist))
        for var in set(os.environ).difference(whitelist):
            del os.environ[var]
        if oebakery.DEBUG:
            debug("Whitelist filtered shell environment:")
            for var in os.environ:
                debug("> %s=%s"%(var, os.environ[var]))
        for var in whitelist:
            if not var in self.config and var in os.environ:
                self.config[var] = os.environ[var]
                debug("importing %s=%s"%(var, os.environ[var]))
        return


    def prepare_cookbook(self):

        self.cookbook = {}

        # collect all available .bb files
        bbrecipes = self.list_bbrecipes()

        # parse all .bb files
        total = len(bbrecipes)
        parsed = 0
        start = datetime.datetime.now()
        for bbrecipe in bbrecipes:
            if not self.options.quiet:
                progress_info("Parsing recipe files", total, parsed)
            data = self.parse_recipe(bbrecipe)
            for extend in data:
                recipe = OEliteRecipe(bbrecipe, extend, data[extend], self.db)
                self.cookbook[recipe.id] = recipe
            parsed += 1
        if not self.options.quiet:
            progress_info("Parsing recipe files", total, parsed)
        if oebakery.DEBUG:
            timing_info("Parsing", start)

        return


    def show(self):

        if len(self.things_todo) == 0:
            die("you must specify something to show")
        elif len(self.things_todo) > 1:
            die("can only show one thing at a time")

        search = self.things_todo[0].split("_", 1)

        if len(search) == 1:
            search.append(None)

        debug("looking for %s"%(repr(search)))
        recipes = self.db.get_recipe_id(name=search[0], version=search[1],
                                        multiple=True)
        debug("recipes=%s %s"%(repr(recipes),
                               repr(self.db.get_recipe(recipes[0]))))

        if not recipes:
            die("no recipe found")
        elif len(recipes) > 1:
            chosen = (recipes[0], self.db.get_recipe(recipes[0])[1])
            for other in recipes[1:]:
                debug("chosen=%s other=%s"%(chosen, other))
                version = self.db.get_recipe(other)[1]
                vercmp = bb.utils.vercmp_part(chosen[1], version)
                if vercmp < 0:
                    chosen = (other, version)
                if vercmp == 0:
                    debug("chosen=%s\nother=%s version=%s"%(chosen, other, version))
                    die("you have to be more precise")
            chosen = chosen[0]
        else:
            chosen = recipes[0]

        recipe = self.cookbook[chosen]

        oelite.data.dump(d=recipe.data, pretty=True,
                         nohash=(not self.options.nohash))

    def bake(self):

        self.setup_tmpdir()

        # task(s) to do
        if self.options.task:
            tasks_todo = self.options.task
        elif "BB_DEFAULT_TASK" in self.config:
            tasks_todo = self.config.getVar("BB_DEFAULT_TASK", 1)
{'fraction_non_alphanumeric': 0.12020171609212706, 'fraction_numerical': 0.005419238295950625, 'mean_word_length': 1.933112582781457, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2087017', 'n_tokens_mistral': 4074, 'n_tokens_neox': 3872, 'n_words': 919}