<commit_before>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import itertools

from conary import callbacks
from conary import changelog
from conary import errors
from conary import versions
from conary.build.nextversion import nextVersion
from conary.deps import deps
from conary.lib import log
from conary.repository import changeset

V_LOADED = 0
V_BREQ = 1
V_REFTRV = 2

# don't change 
DEFAULT_MESSAGE = 1

class ClientClone:

    def _getTroveClosure(self, troveList, callback, fullRecurse, cloneSources):

        seen = set()
        allTroveInfo = set()
        allTroves = dict()
        originalSources = set(troveList)
        toClone = troveList
        while toClone:
            needed = []

            for info in toClone:
                if info[0].startswith("fileset"):
                    raise CloneError, "File sets cannot be cloned"

                if info not in seen:
                    needed.append(info)
                    seen.add(info)

            troves = self.repos.getTroves(needed, withFiles = False, 
                                          callback = callback)
            newToClone = []
            for info, trv in itertools.izip(needed, troves):
                if not trv.getName().endswith(':source'):
                    sourceName = trv.getSourceName()
                    if not sourceName:
                        sourceName = trv.getName().split(':')[0] + ':source'
                    if ':' not in trv.getName() and not fullRecurse:
                        sourcePackage = sourceName.split(':')[0]
                        parentPackage = (sourcePackage, trv.getVersion(),
                                         trv.getFlavor())
                        if parentPackage not in originalSources:
                            # if we're not recursing, we still want to 
                            # clone this as long as the parent package is
                            # the same (this works for groups as well as
                            # for components)
                            continue

                    if cloneSources:
                        sourceTup = (sourceName,
                                     trv.getVersion().getSourceVersion(False),
                                     deps.Flavor())
                        newToClone.append(sourceTup)

                allTroves[info] = trv
                allTroveInfo.add(info)

                newToClone.extend(trv.iterTroveList(strongRefs=True))

            toClone = newToClone
        return allTroves, allTroveInfo


    def createCloneChangeSet(self, targetBranch, troveList = [],
                             updateBuildInfo=True, message=DEFAULT_MESSAGE,
                             infoOnly=False, fullRecurse=False,
                             cloneSources=False, callback=None, trackClone=True):
        if callback is None:
            callback = callbacks.CloneCallback()
        callback.determiningCloneTroves()

        # get the transitive closure
        allTroves, allTroveInfo = self._getTroveClosure(troveList, callback,
            fullRecurse, cloneSources)

        # make sure there are no zeroed timeStamps - targetBranch may be
        # a user-supplied string
        targetBranch = targetBranch.copy()
        targetBranch.resetTimeStamps()

        # split out the binary and sources
        sourceTroveInfo = [ x for x in allTroveInfo 
                                    if x[0].endswith(':source') ]
        binaryTroveInfo = [ x for x in allTroveInfo 
                                    if not x[0].endswith(':source') ]

        versionMap = {}        # maps existing info to the version which is
                               # being cloned by this job, or where that version
                               # has already been cloned to
        leafMap = {}           # maps existing info to the info for the latest
                               # version of that trove on the target branch
        cloneJob = []          # (info, newVersion) tuples

        callback.determiningTargets()
        # start off by finding new version numbers for the sources
        for info in sourceTroveInfo:
            name, version = info[:2]

            try:
                currentVersionList = self.repos.getTroveVersionsByBranch(
                    { name : { targetBranch : None } } )[name].keys()
            except KeyError:
                currentVersionList = []

            if currentVersionList:
                currentVersionList.sort()
                cver = currentVersionList[-1]
                leafMap[info] = (info[0], cver, info[2])

                # if the latest version of the source trove was cloned from the
                # version being cloned, we don't need to reclone the source
                trv = self.repos.getTrove(name, cver, deps.Flavor(),
                    withFiles = False)
                clonedFromVer = trv.troveInfo.clonedFrom()
                if clonedFromVer and clonedFromVer in [version,
                                allTroves[info].troveInfo.clonedFrom()]:
                    # The latest version on the target branch was cloned 
                    # from the same trove the version being cloned was
                    # cloned from
                    # make sure None will not match None
                    versionMap[info] = trv.getVersion()


            if info not in versionMap:
                versionMap[info] = _createSourceVersion(
                            targetBranch, currentVersionList, version)

                cloneJob.append((info, versionMap[info]))

        # now go through the binaries; sort them into buckets based on the
        # source trove each came from. we can't clone troves which came
        # from multiple versions of the same source
        trovesBySource = {}
        for info in binaryTroveInfo:
            trv = allTroves[info]
            source = trv.getSourceName()
            # old troves don't have source info
            assert(source is not None)

            l = trovesBySource.setdefault(trv.getSourceName(), 
                                   (trv.getVersion().getSourceVersion(False), []))
            if l[0] != trv.getVersion().getSourceVersion(False):
                log.error("Clone operation needs multiple versions of %s"
                            % trv.getSourceName())
            l[1].append(info)
            
        # this could be parallelized -- may not be worth the effort though
        for srcTroveName, (sourceVersion, infoList) in \
                                            trovesBySource.iteritems():
            newSourceVersion = versionMap.get(
                    (srcTroveName, sourceVersion, deps.Flavor()), None)
            if newSourceVersion is None:
                # we're not cloning the source at the same time; try and find
                # the source version which was used when the source was cloned
                if targetBranch == sourceVersion.branch():
                    newSourceVersion = sourceVersion
                elif (sourceVersion.isShadow()
                  and not sourceVersion.isModifiedShadow()
                  and sourceVersion.parentVersion().branch() == targetBranch):
                    newSourceVersion = sourceVersion.parentVersion()
                else:
                    try:
                        currentVersionList = \
                            self.repos.getTroveVersionsByBranch(
                              { srcTroveName : { targetBranch : None } } ) \
                                        [srcTroveName].keys()
                    except KeyError:
                        print "No versions of %s exist on branch %s." \
                                    % (srcTroveName, targetBranch.asString()) 
                        return False, None

                    # We are trying to make sure that, if a source exists in
                    # the target branch (while we are cloning binary only), we
                    # match it with the source for the binaries in the
                    # originating branch - it has to be either clonedFrom()
                    # the one on the originating branch, or clonedFrom() the
                    # same trove the source trove on the originating branch
                    # was clonedFrom()

                    # get the trove version for the latest trove on the branch
                    # we try to clone on
                    trv = self.repos.getTrove(srcTroveName, 
                                     currentVersionList[-1],
                                     deps.Flavor(), withFiles = False)

                    clfrom = trv.troveInfo.clonedFrom()
                    srctup = (srcTroveName, sourceVersion, deps.Flavor())
                    if srctup in allTroves:
                        trvcl = allTroves[trvtup]
                    else:
                        # This should not fail, it would mean the upstream
                        # branch has the binaries but not the sources
                        trvcl = self.repos.getTrove(srctup[0], srctup[1],
                                                    srctup[2], withFiles=False)
                    if clfrom and clfrom in [sourceVersion,
                                        trvcl.troveInfo.clonedFrom()]:
                        newSourceVersion = trv.getVersion()
                    else:
                        log.error("Cannot find cloned source for %s=%s" %
                                    (srcTroveName, sourceVersion.asString()))
                        return False, None
                    del currentVersionList

            # we know newSourceVersion is right at this point. now find the new
            # binary version for each flavor
            byFlavor = dict()
            for info in infoList:
                byFlavor.setdefault(info[2], []).append(info)

            for flavor, infoList in byFlavor.iteritems():
                cloneList, newBinaryVersion = \
                            _createBinaryVersions(versionMap, leafMap, 
                                                  self.repos, newSourceVersion, 
                                                  infoList, allTroves, callback)
                versionMap.update(
                    dict((x, newBinaryVersion) for x in cloneList))
                cloneJob += [ (x, newBinaryVersion) for x in cloneList ]
                
        # check versions
        for info, newVersion in cloneJob:
            if not _isUphill(info[1], newVersion) and \
                        not _isSibling(info[1], newVersion):
                log.error("clone only supports cloning troves to parent "
                          "and sibling branches")
                return False, None

        if not cloneJob:
            log.warning("Nothing to clone!")
            return False, None

        allTroves = self.repos.getTroves([ x[0] for x in cloneJob ])

        self._searchExistingClones(targetBranch, cloneJob, allTroves,
            allTroveInfo, updateBuildInfo, versionMap, fullRecurse, callback)


        callback.rewritingFileVersions()

        cs, newFilesNeeded = self._buildChangeSet(cloneJob, allTroves,
            allTroveInfo, versionMap, leafMap, trackClone, updateBuildInfo,
            targetBranch, fullRecurse, infoOnly, callback)

        if infoOnly:
            return True, cs

        callback.gettingCloneData()

        self._addCloneFiles(cs, newFilesNeeded, callback)
        callback.done()

        return True, cs

    def _searchExistingClones(self, targetBranch, cloneJob, allTroves,
                              allTroveInfo, updateBuildInfo, versionMap,
                              fullRecurse, callback):
        needDict = {}
        for (info, newVersion), trv in itertools.izip(cloneJob, allTroves):
            _versionsNeeded(needDict, trv, info[1].branch(), targetBranch,
                            updateBuildInfo, allTroveInfo, fullRecurse)

        for version in versionMap:
            if version in needDict:
                del needDict[version]

        # needDict is indexed by all of the items which don't have versions
        # to map to; we need to look at the target branch and see if there
        # is something good to map to there

        # *** BEGIN search for previously existing clones
        q = {}
        for info in needDict:
            brDict = q.setdefault(info[0], {})
            flList = brDict.setdefault(targetBranch, [])
            flList.append(info[2])

        currentVersions = self.repos.getTroveLeavesByBranch(q, 
                                                            bestFlavor = True)
        matches = []
        for name, verDict in currentVersions.iteritems():
            for version, flavorList in verDict.iteritems():
                matches += [ (name, version, flavor) for flavor in flavorList ]
        # matches is a (name, version, flavor) list of all the troves that
        # could be previous clones of the trove we're cloning now.
        trvs = self.repos.getTroves(matches, withFiles = False, 
                                    callback = callback)
        trvDict = dict(((info[0], info[2]), trv) for (info, trv) in
                            itertools.izip(matches, trvs))
        # troveDict is a (name, flavor) -> trove mapping of things that
        # exist on the target branch

        for info in needDict.keys():
            trv = trvDict.get((info[0], info[2]), None)
            if trv is None:
                continue
            if info[1] in (trv.getVersion(), trv.troveInfo.clonedFrom()):
                versionMap[info] = trv.getVersion()
                # There is a trove of the target label that has a clonedFrom()
                # entry that points to the trove we're trying to clone now.
                # This clone is a duplicate!
                del needDict[info]
        # *** END search for previously existing clones
        _checkNeedsFulfilled(needDict)

        assert(not needDict)

    def _buildChangeSet(self, cloneJob, allTroves, allTroveInfo, versionMap,
                        leafMap, trackClone, updateBuildInfo, targetBranch,
                        fullRecurse, infoOnly, callback):
        cs = changeset.ChangeSet()

        allFilesNeeded = list()

        for (info, newVersion), trv in itertools.izip(cloneJob, allTroves):
            assert(newVersion == versionMap[(trv.getName(), trv.getVersion(),
                                             trv.getFlavor())])

            newVersionHost = newVersion.getHost()
            sourceBranch = info[1].branch()

            # if this is a clone of a clone, use the original clonedFrom value
            # so that all clones refer back to the source-of-all-clones trove
            if trv.troveInfo.clonedFrom() is None and trackClone:
                trv.troveInfo.clonedFrom.set(trv.getVersion())

            # clone the labelPath 
            labelPath = list(trv.getLabelPath())
            labelPath = _computeLabelPath(labelPath,
                                          trv.getVersion().branch().label(),
                                          newVersion.branch().label())
            if labelPath:
                trv.setLabelPath(labelPath)

            trv.changeVersion(newVersion)


            # look through files which aren't already on the right host for
            # inclusion in the change set (this could include too many)
            for (pathId, path, fileId, version) in trv.iterFileList():
                if version.getHost() != newVersionHost:
                    allFilesNeeded.append((pathId, fileId, version))

            needsNewVersions = []
            for (mark, src) in _iterAllVersions(trv, updateBuildInfo):
                if _needsRewrite(sourceBranch, targetBranch, src, mark[0],
                                 allTroveInfo, fullRecurse):
                    _updateVersion(trv, mark, versionMap[src])

            for (pathId, path, fileId, version) in trv.iterFileList():
                if _needsRewrite(sourceBranch, targetBranch, 
                                 (trv.getName(), version, None), None,
                                 allTroveInfo, fullRecurse):
                    needsNewVersions.append((pathId, path, fileId))

            # need to be reversioned
            if needsNewVersions:
                if info in leafMap:
                    oldTrv = self.repos.getTrove(withFiles = True, 
                                                 *leafMap[info])
                    fmap = dict(((x[0], x[2]), x[3]) for x in
                                            oldTrv.iterFileList())
                else:
                    fmap = {}

                for (pathId, path, fileId) in needsNewVersions:
                    ver = fmap.get((pathId, fileId), newVersion)
                    trv.updateFile(pathId, path, ver, fileId)

            if trv.getName().endswith(':source') and not infoOnly:
                try:
                    cl = callback.getCloneChangeLog(trv)
                except:
                    log.error(str(cl))
                    return False, None

                if cl is None:
                    log.error("no change log message was given"
                              " for %s." % trv.getName())
                    return False, None
                trv.changeChangeLog(cl)
            # reset the signatures, because all the versions have now
            # changed, thus invalidating the old sha1 hash
            trv.troveInfo.sigs.reset()
            if not infoOnly: # not computing signatures will make sure this 
                             # doesn't get committed
                trv.computeSignatures()
            trvCs = trv.diff(None, absolute = True)[0]
            cs.newTrove(trvCs)

            if ":" not in trv.getName():
                cs.addPrimaryTrove(trv.getName(), trv.getVersion(), 
                                   trv.getFlavor())

        # the set() removes duplicates
        newFilesNeeded = []
        for (pathId, newFileId, newFileVersion) in set(allFilesNeeded):

            fileHost = newFileVersion.getHost()
            # XXX misa 20070215: newVersionHost set in the previous for loop
            # and used here seems like a bug
            if fileHost == newVersionHost:
                # the file is already present in the repository
                continue

            newFilesNeeded.append((pathId, newFileId, newFileVersion))

        return cs, newFilesNeeded

    def _addCloneFiles(self, cs, newFilesNeeded, callback):<commit_msg>Propagating error / shortcut the rest of the code (CNY-1335)<commit_after>#
# Copyright (c) 2005-2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import itertools

from conary import callbacks
from conary import changelog
from conary import errors
from conary import versions
from conary.build.nextversion import nextVersion
from conary.deps import deps
from conary.lib import log
from conary.repository import changeset

V_LOADED = 0
V_BREQ = 1
V_REFTRV = 2

# don't change 
DEFAULT_MESSAGE = 1

class ClientClone:

    def _getTroveClosure(self, troveList, callback, fullRecurse, cloneSources):

        seen = set()
        allTroveInfo = set()
        allTroves = dict()
        originalSources = set(troveList)
        toClone = troveList
        while toClone:
            needed = []

            for info in toClone:
                if info[0].startswith("fileset"):
                    raise CloneError, "File sets cannot be cloned"

                if info not in seen:
                    needed.append(info)
                    seen.add(info)

            troves = self.repos.getTroves(needed, withFiles = False, 
                                          callback = callback)
            newToClone = []
            for info, trv in itertools.izip(needed, troves):
                if not trv.getName().endswith(':source'):
                    sourceName = trv.getSourceName()
                    if not sourceName:
                        sourceName = trv.getName().split(':')[0] + ':source'
                    if ':' not in trv.getName() and not fullRecurse:
                        sourcePackage = sourceName.split(':')[0]
                        parentPackage = (sourcePackage, trv.getVersion(),
                                         trv.getFlavor())
                        if parentPackage not in originalSources:
                            # if we're not recursing, we still want to 
                            # clone this as long as the parent package is
                            # the same (this works for groups as well as
                            # for components)
                            continue

                    if cloneSources:
                        sourceTup = (sourceName,
                                     trv.getVersion().getSourceVersion(False),
                                     deps.Flavor())
                        newToClone.append(sourceTup)

                allTroves[info] = trv
                allTroveInfo.add(info)

                newToClone.extend(trv.iterTroveList(strongRefs=True))

            toClone = newToClone
        return allTroves, allTroveInfo


    def createCloneChangeSet(self, targetBranch, troveList = [],
                             updateBuildInfo=True, message=DEFAULT_MESSAGE,
                             infoOnly=False, fullRecurse=False,
                             cloneSources=False, callback=None, trackClone=True):
        if callback is None:
            callback = callbacks.CloneCallback()
        callback.determiningCloneTroves()

        # get the transitive closure
        allTroves, allTroveInfo = self._getTroveClosure(troveList, callback,
            fullRecurse, cloneSources)

        # make sure there are no zeroed timeStamps - targetBranch may be
        # a user-supplied string
        targetBranch = targetBranch.copy()
        targetBranch.resetTimeStamps()

        # split out the binary and sources
        sourceTroveInfo = [ x for x in allTroveInfo 
                                    if x[0].endswith(':source') ]
        binaryTroveInfo = [ x for x in allTroveInfo 
                                    if not x[0].endswith(':source') ]

        versionMap = {}        # maps existing info to the version which is
                               # being cloned by this job, or where that version
                               # has already been cloned to
        leafMap = {}           # maps existing info to the info for the latest
                               # version of that trove on the target branch
        cloneJob = []          # (info, newVersion) tuples

        callback.determiningTargets()
        # start off by finding new version numbers for the sources
        for info in sourceTroveInfo:
            name, version = info[:2]

            try:
                currentVersionList = self.repos.getTroveVersionsByBranch(
                    { name : { targetBranch : None } } )[name].keys()
            except KeyError:
                currentVersionList = []

            if currentVersionList:
                currentVersionList.sort()
                cver = currentVersionList[-1]
                leafMap[info] = (info[0], cver, info[2])

                # if the latest version of the source trove was cloned from the
                # version being cloned, we don't need to reclone the source
                trv = self.repos.getTrove(name, cver, deps.Flavor(),
                    withFiles = False)
                clonedFromVer = trv.troveInfo.clonedFrom()
                if clonedFromVer and clonedFromVer in [version,
                                allTroves[info].troveInfo.clonedFrom()]:
                    # The latest version on the target branch was cloned 
                    # from the same trove the version being cloned was
                    # cloned from
                    # make sure None will not match None
                    versionMap[info] = trv.getVersion()


            if info not in versionMap:
                versionMap[info] = _createSourceVersion(
                            targetBranch, currentVersionList, version)

                cloneJob.append((info, versionMap[info]))

        # now go through the binaries; sort them into buckets based on the
        # source trove each came from. we can't clone troves which came
        # from multiple versions of the same source
        trovesBySource = {}
        for info in binaryTroveInfo:
            trv = allTroves[info]
            source = trv.getSourceName()
            # old troves don't have source info
            assert(source is not None)

            l = trovesBySource.setdefault(trv.getSourceName(), 
                                   (trv.getVersion().getSourceVersion(False), []))
            if l[0] != trv.getVersion().getSourceVersion(False):
                log.error("Clone operation needs multiple versions of %s"
                            % trv.getSourceName())
            l[1].append(info)
            
        # this could be parallelized -- may not be worth the effort though
        for srcTroveName, (sourceVersion, infoList) in \
                                            trovesBySource.iteritems():
            newSourceVersion = versionMap.get(
                    (srcTroveName, sourceVersion, deps.Flavor()), None)
            if newSourceVersion is None:
                # we're not cloning the source at the same time; try and find
                # the source version which was used when the source was cloned
                if targetBranch == sourceVersion.branch():
                    newSourceVersion = sourceVersion
                elif (sourceVersion.isShadow()
                  and not sourceVersion.isModifiedShadow()
                  and sourceVersion.parentVersion().branch() == targetBranch):
                    newSourceVersion = sourceVersion.parentVersion()
                else:
                    try:
                        currentVersionList = \
                            self.repos.getTroveVersionsByBranch(
                              { srcTroveName : { targetBranch : None } } ) \
                                        [srcTroveName].keys()
                    except KeyError:
                        print "No versions of %s exist on branch %s." \
                                    % (srcTroveName, targetBranch.asString()) 
                        return False, None

                    # We are trying to make sure that, if a source exists in
                    # the target branch (while we are cloning binary only), we
                    # match it with the source for the binaries in the
                    # originating branch - it has to be either clonedFrom()
                    # the one on the originating branch, or clonedFrom() the
                    # same trove the source trove on the originating branch
                    # was clonedFrom()

                    # get the trove version for the latest trove on the branch
                    # we try to clone on
                    trv = self.repos.getTrove(srcTroveName, 
                                     currentVersionList[-1],
                                     deps.Flavor(), withFiles = False)

                    clfrom = trv.troveInfo.clonedFrom()
                    srctup = (srcTroveName, sourceVersion, deps.Flavor())
                    if srctup in allTroves:
                        trvcl = allTroves[trvtup]
                    else:
                        # This should not fail, it would mean the upstream
                        # branch has the binaries but not the sources
                        trvcl = self.repos.getTrove(srctup[0], srctup[1],
                                                    srctup[2], withFiles=False)
                    if clfrom and clfrom in [sourceVersion,
                                        trvcl.troveInfo.clonedFrom()]:
                        newSourceVersion = trv.getVersion()
                    else:
                        log.error("Cannot find cloned source for %s=%s" %
                                    (srcTroveName, sourceVersion.asString()))
                        return False, None
                    del currentVersionList

            # we know newSourceVersion is right at this point. now find the new
            # binary version for each flavor
            byFlavor = dict()
            for info in infoList:
                byFlavor.setdefault(info[2], []).append(info)

            for flavor, infoList in byFlavor.iteritems():
                cloneList, newBinaryVersion = \
                            _createBinaryVersions(versionMap, leafMap, 
                                                  self.repos, newSourceVersion, 
                                                  infoList, allTroves, callback)
                versionMap.update(
                    dict((x, newBinaryVersion) for x in cloneList))
                cloneJob += [ (x, newBinaryVersion) for x in cloneList ]
                
        # check versions
        for info, newVersion in cloneJob:
            if not _isUphill(info[1], newVersion) and \
                        not _isSibling(info[1], newVersion):
                log.error("clone only supports cloning troves to parent "
                          "and sibling branches")
                return False, None

        if not cloneJob:
            log.warning("Nothing to clone!")
            return False, None

        allTroves = self.repos.getTroves([ x[0] for x in cloneJob ])

        self._searchExistingClones(targetBranch, cloneJob, allTroves,
            allTroveInfo, updateBuildInfo, versionMap, fullRecurse, callback)


        callback.rewritingFileVersions()

        cs, newFilesNeeded = self._buildChangeSet(cloneJob, allTroves,
            allTroveInfo, versionMap, leafMap, trackClone, updateBuildInfo,
            targetBranch, fullRecurse, infoOnly, callback)
        # Propagate empty response
        if cs is None:
            return False, None

        if infoOnly:
            return True, cs

        callback.gettingCloneData()

        self._addCloneFiles(cs, newFilesNeeded, callback)
        callback.done()

        return True, cs

    def _searchExistingClones(self, targetBranch, cloneJob, allTroves,
                              allTroveInfo, updateBuildInfo, versionMap,
                              fullRecurse, callback):
        needDict = {}
        for (info, newVersion), trv in itertools.izip(cloneJob, allTroves):
            _versionsNeeded(needDict, trv, info[1].branch(), targetBranch,
                            updateBuildInfo, allTroveInfo, fullRecurse)

        for version in versionMap:
            if version in needDict:
                del needDict[version]

        # needDict is indexed by all of the items which don't have versions
        # to map to; we need to look at the target branch and see if there
        # is something good to map to there

        # *** BEGIN search for previously existing clones
        q = {}
        for info in needDict:
            brDict = q.setdefault(info[0], {})
            flList = brDict.setdefault(targetBranch, [])
            flList.append(info[2])

        currentVersions = self.repos.getTroveLeavesByBranch(q, 
                                                            bestFlavor = True)
        matches = []
        for name, verDict in currentVersions.iteritems():
            for version, flavorList in verDict.iteritems():
                matches += [ (name, version, flavor) for flavor in flavorList ]
        # matches is a (name, version, flavor) list of all the troves that
        # could be previous clones of the trove we're cloning now.
        trvs = self.repos.getTroves(matches, withFiles = False, 
                                    callback = callback)
        trvDict = dict(((info[0], info[2]), trv) for (info, trv) in
                            itertools.izip(matches, trvs))
        # troveDict is a (name, flavor) -> trove mapping of things that
        # exist on the target branch

        for info in needDict.keys():
            trv = trvDict.get((info[0], info[2]), None)
            if trv is None:
                continue
            if info[1] in (trv.getVersion(), trv.troveInfo.clonedFrom()):
                versionMap[info] = trv.getVersion()
                # There is a trove of the target label that has a clonedFrom()
                # entry that points to the trove we're trying to clone now.
                # This clone is a duplicate!
                del needDict[info]
        # *** END search for previously existing clones
        _checkNeedsFulfilled(needDict)

        assert(not needDict)

    def _buildChangeSet(self, cloneJob, allTroves, allTroveInfo, versionMap,
                        leafMap, trackClone, updateBuildInfo, targetBranch,
                        fullRecurse, infoOnly, callback):
        cs = changeset.ChangeSet()

        allFilesNeeded = list()

        for (info, newVersion), trv in itertools.izip(cloneJob, allTroves):
            assert(newVersion == versionMap[(trv.getName(), trv.getVersion(),
                                             trv.getFlavor())])

            newVersionHost = newVersion.getHost()
            sourceBranch = info[1].branch()

            # if this is a clone of a clone, use the original clonedFrom value
            # so that all clones refer back to the source-of-all-clones trove
            if trv.troveInfo.clonedFrom() is None and trackClone:
                trv.troveInfo.clonedFrom.set(trv.getVersion())

            # clone the labelPath 
            labelPath = list(trv.getLabelPath())
            labelPath = _computeLabelPath(labelPath,
                                          trv.getVersion().branch().label(),
                                          newVersion.branch().label())
            if labelPath:
                trv.setLabelPath(labelPath)

            trv.changeVersion(newVersion)


            # look through files which aren't already on the right host for
            # inclusion in the change set (this could include too many)
            for (pathId, path, fileId, version) in trv.iterFileList():
                if version.getHost() != newVersionHost:
                    allFilesNeeded.append((pathId, fileId, version))

            needsNewVersions = []
            for (mark, src) in _iterAllVersions(trv, updateBuildInfo):
                if _needsRewrite(sourceBranch, targetBranch, src, mark[0],
                                 allTroveInfo, fullRecurse):
                    _updateVersion(trv, mark, versionMap[src])

            for (pathId, path, fileId, version) in trv.iterFileList():
                if _needsRewrite(sourceBranch, targetBranch, 
                                 (trv.getName(), version, None), None,
                                 allTroveInfo, fullRecurse):
                    needsNewVersions.append((pathId, path, fileId))

            # need to be reversioned
            if needsNewVersions:
                if info in leafMap:
                    oldTrv = self.repos.getTrove(withFiles = True, 
                                                 *leafMap[info])
                    fmap = dict(((x[0], x[2]), x[3]) for x in
                                            oldTrv.iterFileList())
                else:
                    fmap = {}

                for (pathId, path, fileId) in needsNewVersions:
                    ver = fmap.get((pathId, fileId), newVersion)
                    trv.updateFile(pathId, path, ver, fileId)

            if trv.getName().endswith(':source') and not infoOnly:
                try:
                    cl = callback.getCloneChangeLog(trv)
                except:
                    log.error(str(cl))
                    return None, None

                if cl is None:
                    log.error("no change log message was given"
                              " for %s." % trv.getName())
                    return None, None
                trv.changeChangeLog(cl)
            # reset the signatures, because all the versions have now
            # changed, thus invalidating the old sha1 hash
            trv.troveInfo.sigs.reset()
            if not infoOnly: # not computing signatures will make sure this 
                             # doesn't get committed
                trv.computeSignatures()
            trvCs = trv.diff(None, absolute = True)[0]
            cs.newTrove(trvCs)

            if ":" not in trv.getName():
                cs.addPrimaryTrove(trv.getName(), trv.getVersion(), 
                                   trv.getFlavor())

        # the set() removes duplicates
        newFilesNeeded = []
        for (pathId, newFileId, newFileVersion) in set(allFilesNeeded):

            fileHost = newFileVersion.getHost()
            # XXX misa 20070215: newVersionHost set in the previous for loop
            # and used here seems like a bug
            if fileHost == newVersionHost:
                # the file is already present in the repository
                continue

            newFilesNeeded.append((pathId, newFileId, newFileVersion))

        return cs, newFilesNeeded

    def _addCloneFiles(self, cs, newFilesNeeded, callback):
{'fraction_non_alphanumeric': 0.07198352278652623, 'fraction_numerical': 0.0030242986755657526, 'mean_word_length': 1.4780024549389494, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 94, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '637601', 'n_tokens_mistral': 9776, 'n_tokens_neox': 9111, 'n_words': 2899}