<commit_before>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import copy
import itertools
import re
import os
import shutil
import tempfile

from conary.build import cook, grouprecipe, use
from conary.build.cook import signAbsoluteChangeset
from conary.conaryclient import cmdline
from conary.deps import deps
from conary.lib import log, magic, util
from conary.repository import trovesource
from conary import checkin
from conary import errors as conaryerrors
from conary import state
from conary import versions

from rmake import errors
from rmake import compat
from rmake.cmdline import cmdutil
from rmake.lib import recipeutil
from rmake.build import buildjob, buildtrove

BUILD_RECURSE_GROUPS_NONE = 0   # don't recurse groups, build the group only
BUILD_RECURSE_GROUPS_BINARY = 1 # find and recurse the binary version of the 
                                # group
BUILD_RECURSE_GROUPS_SOURCE = 2 # find and recurce the source version of the
                                # group

def getBuildJob(buildConfig, conaryclient, troveSpecList,
                message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE, 
                configDict=None):
    trovesByContext = {}

    for troveSpec in list(troveSpecList):
        if not isinstance(troveSpec, tuple):
            troveSpec = cmdutil.parseTroveSpec(troveSpec)

        if len(troveSpec) == 3:
            context = ''
        else:
            context = troveSpec[3]
            troveSpec = troveSpec[:3]

        if troveSpec[2] is None:
            troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))
        trovesByContext.setdefault(context, []).append(troveSpec)

    job = buildjob.BuildJob()

    # don't store all the contexts with this job - they're useless past the
    # initialization step.
    if configDict:
        mainConfig = configDict['']
        job.setMainConfig(configDict[''])
    else:
        cfg = copy.deepcopy(buildConfig)
        cfg.dropContexts()
        mainConfig = cfg
    mainConfig.recurseGroups = int(recurseGroups)
    job.setMainConfig(mainConfig)

    baseMatchRules = mainConfig.matchTroveRule
    for contextStr, troveSpecList in trovesByContext.iteritems():
        if configDict and contextStr in configDict:
            cfg = configDict[contextStr]
        elif contextStr:
            # making this a copy is critical
            cfg = copy.deepcopy(buildConfig)
            for context in contextStr.split(','):
                cfg.setContext(context)
            cfg.dropContexts()
        else:
            cfg = copy.deepcopy(buildConfig)
            cfg.dropContexts()
            contextStr = ''
            job.setMainConfig(cfg)
        cfg.initializeFlavors()
        use.setBuildFlagsFromFlavor(None, cfg.buildFlavor, error=False)
        if not cfg.buildLabel and cfg.installLabelPath:
            cfg.buildLabel = cfg.installLabelPath[0]
        troveSpecList = list(set(troveSpecList))
        troveList = getTrovesToBuild(cfg, conaryclient, troveSpecList,
                         message=None,
                         recurseGroups=recurseGroups,
                         matchSpecs=baseMatchRules + cfg.matchTroveRule)
        for name, version, flavor in troveList:
            if flavor is None:
                flavor = deps.parseFlavor('')
            bt = buildtrove.BuildTrove(None, name, version, flavor,
                                       context=contextStr)
            job.addTrove(name, version, flavor, contextStr, bt)
            job.setTroveConfig(bt, cfg)
    return job

def getTrovesToBuild(cfg, conaryclient, troveSpecList, message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE, matchSpecs=None):
    toBuild = []
    toFind = {}
    groupsToFind = []
    if not matchSpecs:
        matchSpecs = []


    repos = conaryclient.getRepos()
    cfg.resolveTroveTups = _getResolveTroveTups(cfg, repos)
    cfg.recurseGroups = int(recurseGroups)

    cfg.buildTroveSpecs = []
    newTroveSpecs = []
    recipesToCook = []
    for troveSpec in list(troveSpecList):
        if not isinstance(troveSpec, tuple):
            troveSpec = cmdutil.parseTroveSpec(troveSpec)
        if len(troveSpec) == 3:
            context = ''
        else:
            context = troveSpec[3]
            troveSpec = troveSpec[:3]

        if (troveSpec[0].startswith('group-') and not recurseGroups
            and not compat.ConaryVersion().supportsCloneNonRecursive()):
            log.warning('You will not be able to commit this group build'
                        ' without upgrading conary.')
        if troveSpec[2] is None:
            troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))

        if (not troveSpec[1] and not os.path.isdir(troveSpec[0])
            and os.access(troveSpec[0], os.R_OK)):
            # don't rely on cwd, but do allow for symlinks to change
            # when restarting.  Is that sane?  Or should I just do realpath?
            troveSpec[0] == os.path.abspath(troveSpec[0])
            cfg.buildTroveSpecs.append((troveSpec[0], None, troveSpec[2]))
            recipesToCook.append((os.path.realpath(troveSpec[0]), troveSpec[2]))
            continue
        cfg.buildTroveSpecs.append(troveSpec)

        if troveSpec[0].startswith('group-') and recurseGroups:
            groupsToFind.append(troveSpec)
            if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
                newTroveSpecs.append(troveSpec)
        else:
            newTroveSpecs.append(troveSpec)

    localTroves = [(_getLocalCook(conaryclient, x[0], message), x[1])
                     for x in recipesToCook ]
    localTroves = [(x[0][0], x[0][1], x[1]) for x in localTroves]
    if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
        compat.ConaryVersion().requireFindGroupSources()
        localGroupTroves = [ x for x in localTroves 
                             if x[0].startswith('group-') ]
        toBuild.extend(_findSourcesForSourceGroup(repos, cfg, groupsToFind,
                                                  localGroupTroves))
    elif recurseGroups == BUILD_RECURSE_GROUPS_BINARY:
        newTroveSpecs.extend(_findSpecsForBinaryGroup(repos, cfg,
                                                      groupsToFind))

    for troveSpec in newTroveSpecs:
        sourceName = troveSpec[0].split(':')[0] + ':source'

        s = toFind.setdefault((sourceName, troveSpec[1], None), [])
        if troveSpec[2] not in s:
            s.append(troveSpec[2])


    results = repos.findTroves(cfg.buildLabel, toFind, None)

    for troveSpec, troveTups in results.iteritems():
        flavorList = toFind[troveSpec]
        for troveTup in troveTups:
            for flavor in flavorList:
                toBuild.append((troveTup[0], troveTup[1], flavor))

    toBuild.extend(localTroves)

    if matchSpecs:
        toBuild = _filterListByMatchSpecs(cfg.reposName, matchSpecs, toBuild)
    return toBuild

def _filterListByMatchSpecs(reposName, matchSpecs, troveList):
    matchSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=True)
                    for x in matchSpecs ]
    hasAddSpec = False

    troveMap = {}
    for troveTup in troveList:
        key = (troveTup[0].split(':')[0], troveTup[1], troveTup[2])
        troveMap.setdefault(key, []).append(troveTup)

    finalMatchSpecs = {}
    for matchSpec in matchSpecs:
        name = matchSpec[0]
        if name and name[0] == '-':
            removeSpec = True
            name = name[1:]
        else:
            hasAddSpec = True
            removeSpec = False
        if not name:
            finalMatchSpecs.update(dict.fromkeys([(x[0], matchSpec[1],
                                            matchSpec[2]) for x in troveMap],<commit_msg>work w/ reposName being None (RMK-565) affects restart contexts w/ matchRules (RMK-565)<commit_after>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
import copy
import itertools
import re
import os
import shutil
import tempfile

from conary.build import cook, grouprecipe, use
from conary.build.cook import signAbsoluteChangeset
from conary.conaryclient import cmdline
from conary.deps import deps
from conary.lib import log, magic, util
from conary.repository import trovesource
from conary import checkin
from conary import errors as conaryerrors
from conary import state
from conary import versions

from rmake import errors
from rmake import compat
from rmake.cmdline import cmdutil
from rmake.lib import recipeutil
from rmake.build import buildjob, buildtrove

BUILD_RECURSE_GROUPS_NONE = 0   # don't recurse groups, build the group only
BUILD_RECURSE_GROUPS_BINARY = 1 # find and recurse the binary version of the 
                                # group
BUILD_RECURSE_GROUPS_SOURCE = 2 # find and recurce the source version of the
                                # group

def getBuildJob(buildConfig, conaryclient, troveSpecList,
                message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE, 
                configDict=None):
    trovesByContext = {}

    for troveSpec in list(troveSpecList):
        if not isinstance(troveSpec, tuple):
            troveSpec = cmdutil.parseTroveSpec(troveSpec)

        if len(troveSpec) == 3:
            context = ''
        else:
            context = troveSpec[3]
            troveSpec = troveSpec[:3]

        if troveSpec[2] is None:
            troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))
        trovesByContext.setdefault(context, []).append(troveSpec)

    job = buildjob.BuildJob()

    # don't store all the contexts with this job - they're useless past the
    # initialization step.
    if configDict:
        mainConfig = configDict['']
        job.setMainConfig(configDict[''])
    else:
        cfg = copy.deepcopy(buildConfig)
        cfg.dropContexts()
        mainConfig = cfg
    mainConfig.recurseGroups = int(recurseGroups)
    job.setMainConfig(mainConfig)

    baseMatchRules = mainConfig.matchTroveRule
    for contextStr, troveSpecList in trovesByContext.iteritems():
        if configDict and contextStr in configDict:
            cfg = configDict[contextStr]
        elif contextStr:
            # making this a copy is critical
            cfg = copy.deepcopy(buildConfig)
            for context in contextStr.split(','):
                cfg.setContext(context)
            cfg.dropContexts()
        else:
            cfg = copy.deepcopy(buildConfig)
            cfg.dropContexts()
            contextStr = ''
            job.setMainConfig(cfg)
        cfg.initializeFlavors()
        use.setBuildFlagsFromFlavor(None, cfg.buildFlavor, error=False)
        if not cfg.buildLabel and cfg.installLabelPath:
            cfg.buildLabel = cfg.installLabelPath[0]
        troveSpecList = list(set(troveSpecList))
        troveList = getTrovesToBuild(cfg, conaryclient, troveSpecList,
                         message=None,
                         recurseGroups=recurseGroups,
                         matchSpecs=baseMatchRules + cfg.matchTroveRule,
                         reposName=mainConfig.reposName)
        for name, version, flavor in troveList:
            if flavor is None:
                flavor = deps.parseFlavor('')
            bt = buildtrove.BuildTrove(None, name, version, flavor,
                                       context=contextStr)
            job.addTrove(name, version, flavor, contextStr, bt)
            job.setTroveConfig(bt, cfg)
    return job

def getTrovesToBuild(cfg, conaryclient, troveSpecList, message=None, recurseGroups=BUILD_RECURSE_GROUPS_NONE, matchSpecs=None, reposName=None):
    toBuild = []
    toFind = {}
    groupsToFind = []
    if not matchSpecs:
        matchSpecs = []
    if reposName is None:
        reposName = cfg.reposName


    repos = conaryclient.getRepos()
    cfg.resolveTroveTups = _getResolveTroveTups(cfg, repos)
    cfg.recurseGroups = int(recurseGroups)

    cfg.buildTroveSpecs = []
    newTroveSpecs = []
    recipesToCook = []
    for troveSpec in list(troveSpecList):
        if not isinstance(troveSpec, tuple):
            troveSpec = cmdutil.parseTroveSpec(troveSpec)
        if len(troveSpec) == 3:
            context = ''
        else:
            context = troveSpec[3]
            troveSpec = troveSpec[:3]

        if (troveSpec[0].startswith('group-') and not recurseGroups
            and not compat.ConaryVersion().supportsCloneNonRecursive()):
            log.warning('You will not be able to commit this group build'
                        ' without upgrading conary.')
        if troveSpec[2] is None:
            troveSpec = (troveSpec[0], troveSpec[1], deps.parseFlavor(''))

        if (not troveSpec[1] and not os.path.isdir(troveSpec[0])
            and os.access(troveSpec[0], os.R_OK)):
            # don't rely on cwd, but do allow for symlinks to change
            # when restarting.  Is that sane?  Or should I just do realpath?
            troveSpec[0] == os.path.abspath(troveSpec[0])
            cfg.buildTroveSpecs.append((troveSpec[0], None, troveSpec[2]))
            recipesToCook.append((os.path.realpath(troveSpec[0]), troveSpec[2]))
            continue
        cfg.buildTroveSpecs.append(troveSpec)

        if troveSpec[0].startswith('group-') and recurseGroups:
            groupsToFind.append(troveSpec)
            if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
                newTroveSpecs.append(troveSpec)
        else:
            newTroveSpecs.append(troveSpec)

    localTroves = [(_getLocalCook(conaryclient, x[0], message), x[1])
                     for x in recipesToCook ]
    localTroves = [(x[0][0], x[0][1], x[1]) for x in localTroves]
    if recurseGroups == BUILD_RECURSE_GROUPS_SOURCE:
        compat.ConaryVersion().requireFindGroupSources()
        localGroupTroves = [ x for x in localTroves 
                             if x[0].startswith('group-') ]
        toBuild.extend(_findSourcesForSourceGroup(repos, cfg, groupsToFind,
                                                  localGroupTroves))
    elif recurseGroups == BUILD_RECURSE_GROUPS_BINARY:
        newTroveSpecs.extend(_findSpecsForBinaryGroup(repos, cfg,
                                                      groupsToFind))

    for troveSpec in newTroveSpecs:
        sourceName = troveSpec[0].split(':')[0] + ':source'

        s = toFind.setdefault((sourceName, troveSpec[1], None), [])
        if troveSpec[2] not in s:
            s.append(troveSpec[2])


    results = repos.findTroves(cfg.buildLabel, toFind, None)

    for troveSpec, troveTups in results.iteritems():
        flavorList = toFind[troveSpec]
        for troveTup in troveTups:
            for flavor in flavorList:
                toBuild.append((troveTup[0], troveTup[1], flavor))

    toBuild.extend(localTroves)

    if matchSpecs:
        toBuild = _filterListByMatchSpecs(reposName, matchSpecs, toBuild)
    return toBuild

def _filterListByMatchSpecs(reposName, matchSpecs, troveList):
    matchSpecs = [ cmdline.parseTroveSpec(x, allowEmptyName=True)
                    for x in matchSpecs ]
    hasAddSpec = False

    troveMap = {}
    for troveTup in troveList:
        key = (troveTup[0].split(':')[0], troveTup[1], troveTup[2])
        troveMap.setdefault(key, []).append(troveTup)

    finalMatchSpecs = {}
    for matchSpec in matchSpecs:
        name = matchSpec[0]
        if name and name[0] == '-':
            removeSpec = True
            name = name[1:]
        else:
            hasAddSpec = True
            removeSpec = False
        if not name:
            finalMatchSpecs.update(dict.fromkeys([(x[0], matchSpec[1],
                                            matchSpec[2]) for x in troveMap],
{'fraction_non_alphanumeric': 0.08891876320338006, 'fraction_numerical': 0.00806606491261763, 'mean_word_length': 2.3259527357888015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483664', 'n_tokens_mistral': 4858, 'n_tokens_neox': 4633, 'n_words': 1088}