<commit_before>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import copy
import imp
import inspect
import itertools
import new
import os
import string
import sys
import types
import tempfile
import traceback

from conary.repository import errors, trovesource
from conary.build import defaultrecipes
from conary.build import recipe, use
from conary.build import errors as builderrors
from conary.build.errors import RecipeFileError
from conary.build.factory import Factory as FactoryRecipe
from conary.conaryclient import cmdline
from conary.deps import deps
from conary.lib import api, graph, log, util
from conary.local import database
from conary import trove
from conary import versions

class SubloadData(object):

    # Collector for all of the data loadSuperClass and loadInstalled
    # need; this keeps this gunk out of the Importer class directly

    def __init__(self, cfg, repos, db, buildFlavor, directory,
                 branch, name, ignoreInstalled, overrides):
        self.cfg = cfg
        self.repos = repos
        self.db = db
        self.buildFlavor = buildFlavor
        self.parentDir = directory
        self.branch = branch
        self.parentPackageName = name
        self.ignoreInstalled = ignoreInstalled

        if overrides is None:
            self.overrides = {}
        else:
            self.overrides = overrides

class Importer(object):

    baseModuleImports = [
        ('conary.build', ('build', 'action', 'use')),
        ('conary.build.use', ('Arch', 'Use', ('LocalFlags', 'Flags'),
                                            'PackageFlags')),
        ('conary.build.packagerecipe',
                          ('clearBuildReqs', 'clearBuildRequires',
                           'clearCrossReqs', 'clearCrossRequires',
                           'AbstractPackageRecipe',
                           'SourcePackageRecipe',
                           'BaseRequiresRecipe',
                           'PackageRecipe', 'BuildPackageRecipe',
                           'CPackageRecipe', 'AutoPackageRecipe',
                           )),
        ('conary.build.grouprecipe', ('_BaseGroupRecipe', '_GroupRecipe',
                                      'GroupRecipe')),
        ('conary.build.groupsetrecipe', ('_GroupSetRecipe',
                                         'GroupSetRecipe')),
        ('conary.build.filesetrecipe', ('_FilesetRecipe', 'FilesetRecipe')),
        ('conary.build.redirectrecipe', ('_RedirectRecipe', 'RedirectRecipe')),
        ('conary.build.derivedrecipe', ('DerivedChangesetExploder',
                            'AbstractDerivedPackageRecipe',
                            'DerivedPackageRecipe')),
        ('conary.build.inforecipe',  ('UserGroupInfoRecipe',
                                      'UserInfoRecipe', 'GroupInfoRecipe')),
        ('conary.build.capsulerecipe',  ('CapsuleRecipe',
                                      )),
        ('conary.build.derivedcapsulerecipe', ('DerivedChangesetExploder',
                            'AbstractDerivedCapsuleRecipe',
                            'DerivedCapsuleRecipe')),
        ('conary.lib', ('util',)),
        ('os',),
        ('re',),
        ('sys',),
        ('stat',)]
    def __init__(self, objDict = {}, fileName = 'unknownfile.py',
                 baseName = 'unknown', factory = False,
                 subloadData = None):
        self.fileName = fileName
        self.baseName = os.path.basename(self.fileName)
        # can't have a '.' in a module name or import code gets confused
        self.module = imp.new_module(self.baseName.replace('.', '-'))
        self.subloadData = subloadData
        self.loadedTroves = []
        self.loadedSpecs = {}

        for args in self.baseModuleImports:
            self._localImport(*args)

        if factory:
            self._localImport('conary.build.factory',
                              ('Factory', 'FactoryException' ))

        self._copyReusedRecipes()

        if objDict:
            self.module.__dict__.update(objDict.copy())

        self.module.loadInstalled = self.loadInstalled
        self.module.loadSuperClass = self.loadSuperClass
        # XXX when all recipes have been migrated
        # we can get rid of loadRecipe
        self.module.loadRecipe = self.loadSuperClass

    def updateModuleDict(self, d):
        self.module.__dict__.update(d)

    def _copyReusedRecipes(self):
        # XXX HACK - get rid of this when we move the
        # recipe classes to the repository.
        # makes copies of some of the superclass recipes that are
        # created in this module.  (specifically, the ones with buildreqs)
        recipeClassDict = {}
        for recipeClass in self.module.__dict__.values():
            if (type(recipeClass) != type or
                    not issubclass(recipeClass, recipe.Recipe)):
                continue
            numParents = len(inspect.getmro(recipeClass))
            recipeClassDict[recipeClass.__name__] = (numParents, recipeClass)
        # create copies of recipes by the number of parents they have
        # a class always has more parents than its parent does,
        # if you copy the superClasses first, the copies will.
        recipeClasses = [ x[1]  for x in sorted(recipeClassDict.values(),
                                                key=lambda x: x[0]) ]
        for recipeClass in recipeClasses:
            className = recipeClass.__name__
            # when we create a new class object, it needs its superclasses.
            # get the original superclass list and substitute in any
            # copies
            mro = list(inspect.getmro(recipeClass)[1:])
            newMro = []
            for superClass in mro:
                superName = superClass.__name__
                newMro.append(self.module.__dict__.get(superName, superClass))

            newDict = {}
            for name, attr in recipeClass.__dict__.iteritems():
                if type(attr) in [ types.ModuleType, types.MethodType,
                                   types.UnboundMethodType,
                                   types.FunctionType,
                                   staticmethod,
                                   # don't copy in flags, as they
                                   # need to have their data copied out
                                   use.LocalFlagCollection]:
                    newDict[name] = attr
                else:
                    newDict[name] = copy.deepcopy(attr)

            self.module.__dict__[className] = \
                            new.classobj(className, tuple(newMro), newDict)

    def _localImport(self, package, modules=()):
        """
        Import a package into a non-global context.

        @param package: the name of the module to import
        @type package: str
        @param modules: a sequence of modules to import from the package.
        If a 2-tuple is in the sequence, rename the imported module to
        the second value in the tuple.
        @type modules: sequence of strings or tuples, or empty tuple

        Examples of translated import statements::
          from foo import bar as baz:
              _localImport(d, "foo", (("bar", "baz"))
          from bar import fred, george:
              _localImport(d, "bar", ("fred", "george"))
          import os
              _localImport(d, "os")
        """
        m = __import__(package, {}, {}, modules)
        if modules:
            if isinstance(modules, str):
                modules = (modules,)
            for name in modules:
                if type(name) is tuple:
                    mod = name[0]
                    name = name[1]
                else:
                    mod = name
                self.module.__dict__[name] = getattr(m, mod)
        else:
            self.module.__dict__[package] = m
        # save a reference to the module into this context, so it won't
        # be garbage collected until the context is deleted.
        l = self.module.__dict__.setdefault('__localImportModules', [])
        l.append(m)

    def _loadRecipe(self, troveSpec, label, findInstalled):
        """ See docs for loadInstalled and loadSuperClass.  """
        loader = ChainedRecipeLoader(troveSpec, label, findInstalled,
                                     self.subloadData.cfg,
                                     self.subloadData.repos,
                                     self.subloadData.branch,
                                     self.subloadData.parentPackageName,
                                     self.subloadData.parentDir,
                                     self.subloadData.buildFlavor,
                                     self.subloadData.ignoreInstalled,
                                     self.subloadData.overrides,
                                     self.subloadData.db)

        for name, recipe in loader.allRecipes().items():
            # hide all recipes from RecipeLoader - we don't want to return
            # a recipe that has been loaded by loadRecipe, so we treat them
            # for these purposes as if they are abstract base classes
            recipe.internalAbstractBaseClass = 1

            self.module.__dict__[name] = recipe
            if recipe._trove:
                # create a tuple with the version and flavor information needed to
                # load this trove again.   You might be able to rely on the
                # flavor that the trove was built with, but when you load a
                # recipe that is not a superclass of the current recipe,
                # its flavor is not assumed to be relevant to the resulting
                # package (otherwise you might have completely irrelevant flavors
                # showing up for any package that loads the python recipe, e.g.)
                troveTuple = (recipe._trove.getName(), recipe._trove.getVersion(),
                              recipe._usedFlavor)
                log.info('Loaded %s from %s=%s[%s]' % ((name,) + troveTuple))
                self.loadedTroves.extend(loader.getLoadedTroves())
                self.loadedTroves.append(troveTuple)
                self.loadedSpecs[troveSpec] = (troveTuple,
                                               loader.getLoadedSpecs())

        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        self.module.__dict__[loader.recipe.__module__] = loader
        return loader.getRecipe()

    def loadSuperClass(self, troveSpec, label=None):
        """
        Load a recipe so that its class/data can be used as a super class for
        this recipe.

        If the package is not installed anywhere on the system, the C{labelPath}
        will be searched without reference to the installed system.

        @param troveSpec: C{name}I{[}C{=I{version}}I{][}C{[I{flavor}]}I{]}
        specification of the trove to load.  The flavor given will be used
        to find the given recipe and also to set the flavor of the loaded recipe.
        @param label: label string to search for the given recipe in place of
        using the default C{labelPath}.
        If not specified, the labels listed in the version in the including
        recipe will be used as the c{labelPath} to search.
        For example, if called from recipe with version
        C{/conary.rpath.com@rpl:devel//shadow/1.0-1-1},
        the default C{labelPath} that would be constructed would be:
        C{[con<EMAIL>:shadow, conary.rpath.com@rpl:devel]}

        @return: recipe class loaded
        """
        return self._loadRecipe(troveSpec, label, False)

    def loadInstalled(self, troveSpec, label=None):
        """
        Load a recipe so that its data about the installed system can be used
        in this recipe.

        If a complete version is not specified in the trovespec, the version of
        the recipe to load will be based on what is installed on the system.
        For example, if C{loadRecipe('foo')} is called, and package C{foo} with
        version C{/bar.org@bar:devel/4.1-1-1} is installed on the system, then
        C{foo:source} with version C{/bar.org@bar:devel/4.1-1} will be loaded.
        The recipe will also be loaded with the installed package's flavor.

        If the package is not installed anywhere on the system, the C{labelPath}
        will be searched without reference to the installed system.

        @param troveSpec: C{name}I{[}C{=I{version}}I{][}C{[I{flavor}]}I{]}
        specification of the trove to load.  The flavor given will be used
        to find the given recipe and also to set the flavor of the loaded recipe.
        @param label: label string to search for the given recipe in place of
        using the default C{labelPath}.
        If not specified, the labels listed in the version in the including
        recipe will be used as the c{labelPath} to search.
        For example, if called from recipe with version
        C{/conary.rpath.com@rpl:devel//shadow/1.0-1-1},
        the default C{labelPath} that would be constructed would be:
        C{[conary.rpath.com@rpl:shadow, conary.rpath.com@rpl:devel]}

        @return: recipe class loaded
        """
        return self._loadRecipe(troveSpec, label, True)

    def execString(self, codeString):
        try:
            code = compile(codeString, self.fileName, 'exec')
        except SyntaxError, err:
            msg = ('Error in recipe file "%s": %s\n' %(self.baseName, err))
            if err.offset is not None:
                msg += '%s%s^\n' %(err.text, ' ' * (err.offset-1))
            else:
                msg += err.text
            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]

        use.resetUsed()
        try:
            exec code in self.module.__dict__
        except (errors.ConaryError, builderrors.CvcError), err:
            # don't show the exception for conary and cvc errors -
            # we assume their exception message already contains the
            # required information

            tb = sys.exc_info()[2]
            while tb and tb.tb_frame.f_code.co_filename != self.fileName:
                tb = tb.tb_next
            linenum = tb.tb_frame.f_lineno

            msg = ('Error in recipe file "%s", line %s:\n %s' %
                                (self.baseName, linenum, err))


            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]
        except Exception, err:
            tb = sys.exc_info()[2]
            while tb and tb.tb_frame.f_code.co_filename != self.fileName:
                tb = tb.tb_next

            if not tb:
                raise

            err = ''.join(traceback.format_exception(err.__class__, err, tb))
            del tb
            msg = ('Error in recipe file "%s":\n %s' %(self.baseName, err))
            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]

class RecipeLoaderFromString(object):

    loadedTroves = None
    loadedSpecs = None
    cookType = recipe.Recipe.COOK_TYPE_LOCAL

    def __init__(self, codeString, filename, cfg=None, repos=None,
                 component=None, branch=None, ignoreInstalled=False,
                 directory=None, buildFlavor=None, db=None, overrides = None,
                 factory = False, objDict = {}, loadAutoRecipes = True):
        try:
            self._load(codeString, filename, cfg, repos, component,
                       branch, ignoreInstalled, directory,
                       buildFlavor=buildFlavor, db=db,
                       overrides=overrides, factory = factory,
                       objDict = objDict, loadAutoRecipes = loadAutoRecipes)
        except Exception, err:
            raise builderrors.LoadRecipeError, \
                    'unable to load recipe file %s:\n%s' % (filename, err), \
                    sys.exc_info()[2]

    @staticmethod
    def _loadAutoRecipes(importer, cfg, repos, db = None, buildFlavor = None):
        def _loadTroves(repos, nvfDict, troveSpecStrs, troveSpecs):
            """
            Loads troves from the repository after they've been found
            """
            trovesNeeded = []
            for i, (specStr, spec) in \
                        enumerate(itertools.izip(troveSpecStrs, troveSpecs)):
                nvf = nvfDict.get(spec, None)
                if not nvf:
                    raise builderrors.RecipeFileError('no match for '
                                    'autoLoadRecipe entry %s' % specStr)

                if len(nvf) > 1:
                    raise builderrors.RecipeFileError('too many matches for '
                                    'autoLoadRecipe entry %s' % specStr)

                trovesNeeded.append((i, nvf[0]))

            troves = repos.getTroves([ x[1] for x in trovesNeeded],
                                     withFiles = False)

            result = [ None ] * len(troveSpecs)
            for ((i, nvf), trv) in itertools.izip(trovesNeeded, troves):
                result[i] = trv

            return result

        # def _loadDefaultPackages begins here

        if not cfg.autoLoadRecipes:
            return

        RecipeLoaderFromString._loadingDefaults = True

        if db is None:
            db = database.Database(cfg.root, cfg.dbPath)

        # This stack looks in the database before looking at a repository,
        # avoiding repository calls where they aren't needed.
        ts = trovesource.stack(db, repos)

        troveSpecs = [ cmdline.parseTroveSpec(x) for x in cfg.autoLoadRecipes ]

        # Look on the repository first to match the trove specs
        try:
            nvfDict = repos.findTroves(cfg.installLabelPath, troveSpecs,
                                       cfg.flavor,
                                       allowMissing=True,
                                       bestFlavor=True)
        except errors.OpenError, err:
            nvfDict = {}

        neededTroveSpecs = [ x for x in troveSpecs if x not in nvfDict ]
        nvfDict.update(db.findTroves(cfg.installLabelPath, neededTroveSpecs,
                                     allowMissing = True))

        groupTroves = _loadTroves(ts, nvfDict, cfg.autoLoadRecipes, troveSpecs)

        # We look for recipes in reverse order to allow the troves at the
        # front of the list to override those at the end
        recipeTroves = {}

        for trv in reversed(groupTroves):
            for x in itertools.chain([ trv.getNameVersionFlavor() ],
                                      trv.iterTroveList(weakRefs = True,
                                                        strongRefs = True) ):
                if x[0].endswith(':recipe'):
                    recipeTroves[x[0]] = x<commit_msg>loadrecipe: relax the flavor matching used to find autoLoadRecipes<commit_after>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import copy
import imp
import inspect
import itertools
import new
import os
import string
import sys
import types
import tempfile
import traceback

from conary.repository import errors
from conary.build import recipe, use
from conary.build import errors as builderrors
from conary.build.errors import RecipeFileError
from conary.conaryclient import cmdline
from conary.deps import deps
from conary.lib import api, graph, log, util
from conary.local import database
from conary.repository import searchsource
from conary.repository import trovesource
from conary import versions

class SubloadData(object):

    # Collector for all of the data loadSuperClass and loadInstalled
    # need; this keeps this gunk out of the Importer class directly

    def __init__(self, cfg, repos, db, buildFlavor, directory,
                 branch, name, ignoreInstalled, overrides):
        self.cfg = cfg
        self.repos = repos
        self.db = db
        self.buildFlavor = buildFlavor
        self.parentDir = directory
        self.branch = branch
        self.parentPackageName = name
        self.ignoreInstalled = ignoreInstalled

        if overrides is None:
            self.overrides = {}
        else:
            self.overrides = overrides

class Importer(object):

    baseModuleImports = [
        ('conary.build', ('build', 'action', 'use')),
        ('conary.build.use', ('Arch', 'Use', ('LocalFlags', 'Flags'),
                                            'PackageFlags')),
        ('conary.build.packagerecipe',
                          ('clearBuildReqs', 'clearBuildRequires',
                           'clearCrossReqs', 'clearCrossRequires',
                           'AbstractPackageRecipe',
                           'SourcePackageRecipe',
                           'BaseRequiresRecipe',
                           'PackageRecipe', 'BuildPackageRecipe',
                           'CPackageRecipe', 'AutoPackageRecipe',
                           )),
        ('conary.build.grouprecipe', ('_BaseGroupRecipe', '_GroupRecipe',
                                      'GroupRecipe')),
        ('conary.build.groupsetrecipe', ('_GroupSetRecipe',
                                         'GroupSetRecipe')),
        ('conary.build.filesetrecipe', ('_FilesetRecipe', 'FilesetRecipe')),
        ('conary.build.redirectrecipe', ('_RedirectRecipe', 'RedirectRecipe')),
        ('conary.build.derivedrecipe', ('DerivedChangesetExploder',
                            'AbstractDerivedPackageRecipe',
                            'DerivedPackageRecipe')),
        ('conary.build.inforecipe',  ('UserGroupInfoRecipe',
                                      'UserInfoRecipe', 'GroupInfoRecipe')),
        ('conary.build.capsulerecipe',  ('CapsuleRecipe',
                                      )),
        ('conary.build.derivedcapsulerecipe', ('DerivedChangesetExploder',
                            'AbstractDerivedCapsuleRecipe',
                            'DerivedCapsuleRecipe')),
        ('conary.lib', ('util',)),
        ('os',),
        ('re',),
        ('sys',),
        ('stat',)]
    def __init__(self, objDict = {}, fileName = 'unknownfile.py',
                 baseName = 'unknown', factory = False,
                 subloadData = None):
        self.fileName = fileName
        self.baseName = os.path.basename(self.fileName)
        # can't have a '.' in a module name or import code gets confused
        self.module = imp.new_module(self.baseName.replace('.', '-'))
        self.subloadData = subloadData
        self.loadedTroves = []
        self.loadedSpecs = {}

        for args in self.baseModuleImports:
            self._localImport(*args)

        if factory:
            self._localImport('conary.build.factory',
                              ('Factory', 'FactoryException' ))

        self._copyReusedRecipes()

        if objDict:
            self.module.__dict__.update(objDict.copy())

        self.module.loadInstalled = self.loadInstalled
        self.module.loadSuperClass = self.loadSuperClass
        # XXX when all recipes have been migrated
        # we can get rid of loadRecipe
        self.module.loadRecipe = self.loadSuperClass

    def updateModuleDict(self, d):
        self.module.__dict__.update(d)

    def _copyReusedRecipes(self):
        # XXX HACK - get rid of this when we move the
        # recipe classes to the repository.
        # makes copies of some of the superclass recipes that are
        # created in this module.  (specifically, the ones with buildreqs)
        recipeClassDict = {}
        for recipeClass in self.module.__dict__.values():
            if (type(recipeClass) != type or
                    not issubclass(recipeClass, recipe.Recipe)):
                continue
            numParents = len(inspect.getmro(recipeClass))
            recipeClassDict[recipeClass.__name__] = (numParents, recipeClass)
        # create copies of recipes by the number of parents they have
        # a class always has more parents than its parent does,
        # if you copy the superClasses first, the copies will.
        recipeClasses = [ x[1]  for x in sorted(recipeClassDict.values(),
                                                key=lambda x: x[0]) ]
        for recipeClass in recipeClasses:
            className = recipeClass.__name__
            # when we create a new class object, it needs its superclasses.
            # get the original superclass list and substitute in any
            # copies
            mro = list(inspect.getmro(recipeClass)[1:])
            newMro = []
            for superClass in mro:
                superName = superClass.__name__
                newMro.append(self.module.__dict__.get(superName, superClass))

            newDict = {}
            for name, attr in recipeClass.__dict__.iteritems():
                if type(attr) in [ types.ModuleType, types.MethodType,
                                   types.UnboundMethodType,
                                   types.FunctionType,
                                   staticmethod,
                                   # don't copy in flags, as they
                                   # need to have their data copied out
                                   use.LocalFlagCollection]:
                    newDict[name] = attr
                else:
                    newDict[name] = copy.deepcopy(attr)

            self.module.__dict__[className] = \
                            new.classobj(className, tuple(newMro), newDict)

    def _localImport(self, package, modules=()):
        """
        Import a package into a non-global context.

        @param package: the name of the module to import
        @type package: str
        @param modules: a sequence of modules to import from the package.
        If a 2-tuple is in the sequence, rename the imported module to
        the second value in the tuple.
        @type modules: sequence of strings or tuples, or empty tuple

        Examples of translated import statements::
          from foo import bar as baz:
              _localImport(d, "foo", (("bar", "baz"))
          from bar import fred, george:
              _localImport(d, "bar", ("fred", "george"))
          import os
              _localImport(d, "os")
        """
        m = __import__(package, {}, {}, modules)
        if modules:
            if isinstance(modules, str):
                modules = (modules,)
            for name in modules:
                if type(name) is tuple:
                    mod = name[0]
                    name = name[1]
                else:
                    mod = name
                self.module.__dict__[name] = getattr(m, mod)
        else:
            self.module.__dict__[package] = m
        # save a reference to the module into this context, so it won't
        # be garbage collected until the context is deleted.
        l = self.module.__dict__.setdefault('__localImportModules', [])
        l.append(m)

    def _loadRecipe(self, troveSpec, label, findInstalled):
        """ See docs for loadInstalled and loadSuperClass.  """
        loader = ChainedRecipeLoader(troveSpec, label, findInstalled,
                                     self.subloadData.cfg,
                                     self.subloadData.repos,
                                     self.subloadData.branch,
                                     self.subloadData.parentPackageName,
                                     self.subloadData.parentDir,
                                     self.subloadData.buildFlavor,
                                     self.subloadData.ignoreInstalled,
                                     self.subloadData.overrides,
                                     self.subloadData.db)

        for name, recipe in loader.allRecipes().items():
            # hide all recipes from RecipeLoader - we don't want to return
            # a recipe that has been loaded by loadRecipe, so we treat them
            # for these purposes as if they are abstract base classes
            recipe.internalAbstractBaseClass = 1

            self.module.__dict__[name] = recipe
            if recipe._trove:
                # create a tuple with the version and flavor information needed to
                # load this trove again.   You might be able to rely on the
                # flavor that the trove was built with, but when you load a
                # recipe that is not a superclass of the current recipe,
                # its flavor is not assumed to be relevant to the resulting
                # package (otherwise you might have completely irrelevant flavors
                # showing up for any package that loads the python recipe, e.g.)
                troveTuple = (recipe._trove.getName(), recipe._trove.getVersion(),
                              recipe._usedFlavor)
                log.info('Loaded %s from %s=%s[%s]' % ((name,) + troveTuple))
                self.loadedTroves.extend(loader.getLoadedTroves())
                self.loadedTroves.append(troveTuple)
                self.loadedSpecs[troveSpec] = (troveTuple,
                                               loader.getLoadedSpecs())

        # stash a reference to the module in the namespace
        # of the recipe that loaded it, or else it will be destroyed
        self.module.__dict__[loader.recipe.__module__] = loader
        return loader.getRecipe()

    def loadSuperClass(self, troveSpec, label=None):
        """
        Load a recipe so that its class/data can be used as a super class for
        this recipe.

        If the package is not installed anywhere on the system, the C{labelPath}
        will be searched without reference to the installed system.

        @param troveSpec: C{name}I{[}C{=I{version}}I{][}C{[I{flavor}]}I{]}
        specification of the trove to load.  The flavor given will be used
        to find the given recipe and also to set the flavor of the loaded recipe.
        @param label: label string to search for the given recipe in place of
        using the default C{labelPath}.
        If not specified, the labels listed in the version in the including
        recipe will be used as the c{labelPath} to search.
        For example, if called from recipe with version
        C{/conary.rpath.com@rpl:devel//shadow/1.0-1-1},
        the default C{labelPath} that would be constructed would be:
        C{[conary.rpath.com@rpl:shadow, conary.rpath.com@rpl:devel]}

        @return: recipe class loaded
        """
        return self._loadRecipe(troveSpec, label, False)

    def loadInstalled(self, troveSpec, label=None):
        """
        Load a recipe so that its data about the installed system can be used
        in this recipe.

        If a complete version is not specified in the trovespec, the version of
        the recipe to load will be based on what is installed on the system.
        For example, if C{loadRecipe('foo')} is called, and package C{foo} with
        version C{/bar.org@bar:devel/4.1-1-1} is installed on the system, then
        C{foo:source} with version C{/bar.org@bar:devel/4.1-1} will be loaded.
        The recipe will also be loaded with the installed package's flavor.

        If the package is not installed anywhere on the system, the C{labelPath}
        will be searched without reference to the installed system.

        @param troveSpec: C{name}I{[}C{=I{version}}I{][}C{[I{flavor}]}I{]}
        specification of the trove to load.  The flavor given will be used
        to find the given recipe and also to set the flavor of the loaded recipe.
        @param label: label string to search for the given recipe in place of
        using the default C{labelPath}.
        If not specified, the labels listed in the version in the including
        recipe will be used as the c{labelPath} to search.
        For example, if called from recipe with version
        C{/conary.rpath.com@rpl:devel//shadow/1.0-1-1},
        the default C{labelPath} that would be constructed would be:
        C{[conary.rpath.com@rpl:shadow, conary.rpath.com@rpl:devel]}

        @return: recipe class loaded
        """
        return self._loadRecipe(troveSpec, label, True)

    def execString(self, codeString):
        try:
            code = compile(codeString, self.fileName, 'exec')
        except SyntaxError, err:
            msg = ('Error in recipe file "%s": %s\n' %(self.baseName, err))
            if err.offset is not None:
                msg += '%s%s^\n' %(err.text, ' ' * (err.offset-1))
            else:
                msg += err.text
            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]

        use.resetUsed()
        try:
            exec code in self.module.__dict__
        except (errors.ConaryError, builderrors.CvcError), err:
            # don't show the exception for conary and cvc errors -
            # we assume their exception message already contains the
            # required information

            tb = sys.exc_info()[2]
            while tb and tb.tb_frame.f_code.co_filename != self.fileName:
                tb = tb.tb_next
            linenum = tb.tb_frame.f_lineno

            msg = ('Error in recipe file "%s", line %s:\n %s' %
                                (self.baseName, linenum, err))


            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]
        except Exception, err:
            tb = sys.exc_info()[2]
            while tb and tb.tb_frame.f_code.co_filename != self.fileName:
                tb = tb.tb_next

            if not tb:
                raise

            err = ''.join(traceback.format_exception(err.__class__, err, tb))
            del tb
            msg = ('Error in recipe file "%s":\n %s' %(self.baseName, err))
            raise builderrors.RecipeFileError, msg, sys.exc_info()[2]

class RecipeLoaderFromString(object):

    loadedTroves = None
    loadedSpecs = None
    cookType = recipe.Recipe.COOK_TYPE_LOCAL

    def __init__(self, codeString, filename, cfg=None, repos=None,
                 component=None, branch=None, ignoreInstalled=False,
                 directory=None, buildFlavor=None, db=None, overrides = None,
                 factory = False, objDict = {}, loadAutoRecipes = True):
        try:
            self._load(codeString, filename, cfg, repos, component,
                       branch, ignoreInstalled, directory,
                       buildFlavor=buildFlavor, db=db,
                       overrides=overrides, factory = factory,
                       objDict = objDict, loadAutoRecipes = loadAutoRecipes)
        except Exception, err:
            raise builderrors.LoadRecipeError, \
                    'unable to load recipe file %s:\n%s' % (filename, err), \
                    sys.exc_info()[2]

    @staticmethod
    def _loadAutoRecipes(importer, cfg, repos, db = None, buildFlavor = None):
        def _loadTroves(repos, nvfDict, troveSpecStrs, troveSpecs):
            """
            Loads troves from the repository after they've been found
            """
            trovesNeeded = []
            for i, (specStr, spec) in \
                        enumerate(itertools.izip(troveSpecStrs, troveSpecs)):
                nvf = nvfDict.get(spec, None)
                if not nvf:
                    raise builderrors.RecipeFileError('no match for '
                                    'autoLoadRecipe entry %s' % specStr)

                if len(nvf) > 1:
                    raise builderrors.RecipeFileError('too many matches for '
                                    'autoLoadRecipe entry %s' % specStr)

                trovesNeeded.append((i, nvf[0]))

            troves = repos.getTroves([ x[1] for x in trovesNeeded],
                                     withFiles = False)

            result = [ None ] * len(troveSpecs)
            for ((i, nvf), trv) in itertools.izip(trovesNeeded, troves):
                result[i] = trv

            return result

        # def _loadDefaultPackages begins here

        if not cfg.autoLoadRecipes:
            return

        RecipeLoaderFromString._loadingDefaults = True

        if db is None:
            db = database.Database(cfg.root, cfg.dbPath)

        # This stack looks in the database before looking at a repository,
        # avoiding repository calls where they aren't needed.
        ts = trovesource.stack(db, repos)

        troveSpecs = [ cmdline.parseTroveSpec(x) for x in cfg.autoLoadRecipes ]

        # Look on the repository first to match the trove specs
        searchSource = searchsource.NetworkSearchSource(repos,
                cfg.installLabelPath, cfg.flavor, db)
        try:
            nvfDict = searchSource.findTroves(troveSpecs, allowMissing=True,
                    bestFlavor=True)
        except errors.OpenError, err:
            nvfDict = {}

        neededTroveSpecs = [ x for x in troveSpecs if x not in nvfDict ]
        nvfDict.update(db.findTroves(cfg.installLabelPath, neededTroveSpecs,
                                     allowMissing = True))

        groupTroves = _loadTroves(ts, nvfDict, cfg.autoLoadRecipes, troveSpecs)

        # We look for recipes in reverse order to allow the troves at the
        # front of the list to override those at the end
        recipeTroves = {}

        for trv in reversed(groupTroves):
            for x in itertools.chain([ trv.getNameVersionFlavor() ],
                                      trv.iterTroveList(weakRefs = True,
                                                        strongRefs = True) ):
                if x[0].endswith(':recipe'):
                    recipeTroves[x[0]] = x
{'fraction_non_alphanumeric': 0.08370846452082954, 'fraction_numerical': 0.0019775701907314407, 'mean_word_length': 1.808330288637194, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 71, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '632934', 'n_tokens_mistral': 10211, 'n_tokens_neox': 9856, 'n_words': 3153}