<commit_before>#!/usr/bin/python2.4
# -*- mode: python -*-
#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
#

import itertools
import os

from conary import conarycfg
from conary.lib import cfg, util, log

class MirrorConfigurationSection(cfg.ConfigSection):
    repositoryMap         =  conarycfg.CfgRepoMap
    user                  =  conarycfg.CfgUserInfo

class MirrorConfiguration(cfg.SectionedConfigFile):
    host                  =  cfg.CfgString
    entitlementDirectory  =  (cfg.CfgPath, '/etc/conary/entitlements')
    labels                =  conarycfg.CfgInstallLabelPath
    source                =  MirrorConfigurationSection
    target                =  MirrorConfigurationSection

    def __init__(self):
        cfg.SectionedConfigFile.__init__(self)


def filterAlreadyPresent(repos, troveList):
    # Filter out troves which are already in the local repository. Since
    # the marks aren't distinct (they increase, but not monotonially), it's
    # possible that something new got committed with the same mark we
    # last updated to, so we have to look again at all of the troves in the
    # source repository with the last mark which made it into our target.
    present = repos.hasTroves([ x[1] for x in troveList ])
    filteredList = [ x for x in troveList if not present[x[1]] ]

    return filteredList

def filterSigsWithoutTroves(repos, currentMark, sigList):
    # Sigs whose mark is the same as currentMark might not have their trove
    # available on the server (it might be coming as part of this mirror
    # run). sigList has to be sorted by mark for this to work.

    inQuestion = [ x for x in sigList if x[0] == currentMark ]
    present = repos.hasTroves([ x[1] for x in inQuestion ])
    # keep troves whose mark are older than currentMark and troves which
    # are present on the target
    sigList = [ x for x in sigList if present.get(x[1], True) ]
    return sigList

def groupTroves(troveList):
    # combine the troves into indisolvable groups based on their version and
    # flavor; it's assumed that adjacent troves with the same version/flavor
    # must be in a single commit
    grouping = {}
    for info in troveList:
        (n, v, f) = info[1]
        crtGrp = grouping.setdefault((v,f), [])
        crtGrp.append(info)
    return grouping.values()

def buildJobList(repos, groupList):
    # Match each trove with something we already have; this is to mirror
    # using relative changesets, which is a lot more efficient than using
    # absolute ones.
    q = {}
    for group in groupList:
        for mark, (name, version, flavor) in group:
            d = q.setdefault(name, {})
            l = d.setdefault(version.branch(), [])
            l.append(flavor)

    latestAvailable = repos.getTroveLeavesByBranch(q)

    # we'll keep latestAvailable in sync with what the target will look like
    # as the mirror progresses
    jobList = []
    for group in groupList:
        groupJobList = []
        # for each job find what it's relative to and build up groupJobList
        # as the job list for this group
        for mark, (name, version, flavor) in group:
            if name not in latestAvailable:
                job = (name, (None, None), (version, flavor), True)
                currentMatch = (None, None, None, None)
            else:
                d = latestAvailable[name]
                verFlvMap = set()
                # name, version, versionDistance, flavorScore
                currentMatch = (None, None, None, None)
                for repVersion, flavorList in d.iteritems():
                    for repFlavor in flavorList:
                        score = flavor.score(repFlavor)
                        if repVersion == version:
                            closeness = 100000
                        else:
                            closeness = version.closeness(repVersion)

                        if score is False: continue
                        if score < currentMatch[3]:
                            continue
                        elif score > currentMatch[3]:
                            currentMatch = (repVersion, repFlavor, closeness,
                                            score)
                        elif closeness < currentMatch[2]:
                            continue
                        else:
                            currentMatch = (repVersion, repFlavor, closeness,
                                            score)

                job = (name, (currentMatch[0], currentMatch[1]),
                              (version, flavor), currentMatch[0] is None)

            groupJobList.append((mark, job))

        # now iterate through groupJobList and update latestAvailable to
        # reflect the state of the mirror after this job completes
        for mark, job in groupJobList:
            name = job[0]
            oldVersion, oldFlavor = job[1]
            newVersion, newFlavor = job[2]

            d = latestAvailable.setdefault(name, {})

            if oldVersion in d and oldVersion.branch() == newVersion.branch():
                # If the old version is on the same branch as the new one,
                # replace the old with the new. If it's on a different
                # branch, we'll track both.
                d[oldVersion].remove(oldFlavor)
                if not d[oldVersion]: del d[oldVersion]

            flavorList = d.setdefault(newVersion, [])
            flavorList.append(newFlavor)

        jobList.append(groupJobList)

    return jobList

def mirrorSignatures(sourceRepos, targetRepos, currentMark, cfg,
                     test = False, syncSigs = False):
    log.debug("looking for new pgp keys")
    keyList = sourceRepos.getNewPGPKeys(cfg.host, currentMark)
    if test:
        log.debug("(not adding %d keys due to test mode)", len(keyList))
    else:
        log.debug("adding %d keys to target", len(keyList))
        targetRepos.addPGPKeyList(cfg.host, keyList)

    if syncSigs:
        log.debug("getting full trove list for signature sync")
        troveDict = sourceRepos.getTroveVersionList(cfg.host, { None : None })
        sigList = []
        for name, versionD in troveDict.iterkeys():
            for version, flavorList in versionD.iterkeys():
                sigList += [ (name, version, x) for x in flavorList ]
    else:
        log.debug("looking for new trove signatures")
        sigList = sourceRepos.getNewSigList(cfg.host, currentMark)
    # protection against duplicate items returned in the list by some servers
    sigList = list(set(sigList))
    sigList.sort(lambda a,b: cmp(a[0], b[0]))
    log.debug("%d new signatures are available" % len(sigList))

    # also weed out the signatures that don't belong on our label. Having none
    # left after this isn't that big a deal, so we  don't have to return
    if cfg.labels and len(sigList):
        sigList = [ x for x in sigList if
                            x[1][1].branch().label() in cfg.labels ]
        log.debug("after label filtering %d sigs are needed", len(sigList))

    log.debug("removing signatures for troves not yet mirrored")
    sigList = filterSigsWithoutTroves(targetRepos, currentMark, sigList)
    log.debug("%d signatures need to be mirrored", len(sigList))

    updateCount = 0
    if sigList:
        sigs = sourceRepos.getTroveSigs([ x[1] for x in sigList ])
        if test:
            log.debug("not mirroring %d signatures due to test mode", len(sigs))
        else:
            log.debug("mirroring %d sigs", len(sigs))
            updateCount = targetRepos.setTroveSigs(
                [ (x[0][1], x[1]) for x in itertools.izip(sigList, sigs) ])
    return updateCount

def mirrorRepository(sourceRepos, targetRepos, cfg,
                     test = False, sync = False, syncSigs = False):
    # find the latest timestamp stored on the target mirror
    if sync:
        currentMark = -1
        targetRepos.setMirrorMark(cfg.host, currentMark)
    else:
        currentMark = targetRepos.getMirrorMark(cfg.host)
    log.debug("currently up to date through %d", int(currentMark))

    # first, mirror signatures for troves already mirrored
    updateCount = mirrorSignatures(sourceRepos, targetRepos, currentMark,
                                   cfg = cfg, test = test, syncSigs = syncSigs)

    # now find all of the troves we need from from the mirror source<commit_msg>fix the client side of mirroring with --full-sig-sync<commit_after>#!/usr/bin/python2.4
# -*- mode: python -*-
#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
#

import itertools
import os

from conary import conarycfg
from conary.lib import cfg, util, log

class MirrorConfigurationSection(cfg.ConfigSection):
    repositoryMap         =  conarycfg.CfgRepoMap
    user                  =  conarycfg.CfgUserInfo

class MirrorConfiguration(cfg.SectionedConfigFile):
    host                  =  cfg.CfgString
    entitlementDirectory  =  (cfg.CfgPath, '/etc/conary/entitlements')
    labels                =  conarycfg.CfgInstallLabelPath
    source                =  MirrorConfigurationSection
    target                =  MirrorConfigurationSection

    def __init__(self):
        cfg.SectionedConfigFile.__init__(self)


def filterAlreadyPresent(repos, troveList):
    # Filter out troves which are already in the local repository. Since
    # the marks aren't distinct (they increase, but not monotonially), it's
    # possible that something new got committed with the same mark we
    # last updated to, so we have to look again at all of the troves in the
    # source repository with the last mark which made it into our target.
    present = repos.hasTroves([ x[1] for x in troveList ])
    filteredList = [ x for x in troveList if not present[x[1]] ]

    return filteredList

def filterSigsWithoutTroves(repos, currentMark, sigList):
    # Sigs whose mark is the same as currentMark might not have their trove
    # available on the server (it might be coming as part of this mirror
    # run). sigList has to be sorted by mark for this to work.

    inQuestion = [ x for x in sigList if x[0] == currentMark ]
    present = repos.hasTroves([ x[1] for x in inQuestion ])
    # keep troves whose mark are older than currentMark and troves which
    # are present on the target
    sigList = [ x for x in sigList if present.get(x[1], True) ]
    return sigList

def groupTroves(troveList):
    # combine the troves into indisolvable groups based on their version and
    # flavor; it's assumed that adjacent troves with the same version/flavor
    # must be in a single commit
    grouping = {}
    for info in troveList:
        (n, v, f) = info[1]
        crtGrp = grouping.setdefault((v,f), [])
        crtGrp.append(info)
    return grouping.values()

def buildJobList(repos, groupList):
    # Match each trove with something we already have; this is to mirror
    # using relative changesets, which is a lot more efficient than using
    # absolute ones.
    q = {}
    for group in groupList:
        for mark, (name, version, flavor) in group:
            d = q.setdefault(name, {})
            l = d.setdefault(version.branch(), [])
            l.append(flavor)

    latestAvailable = repos.getTroveLeavesByBranch(q)

    # we'll keep latestAvailable in sync with what the target will look like
    # as the mirror progresses
    jobList = []
    for group in groupList:
        groupJobList = []
        # for each job find what it's relative to and build up groupJobList
        # as the job list for this group
        for mark, (name, version, flavor) in group:
            if name not in latestAvailable:
                job = (name, (None, None), (version, flavor), True)
                currentMatch = (None, None, None, None)
            else:
                d = latestAvailable[name]
                verFlvMap = set()
                # name, version, versionDistance, flavorScore
                currentMatch = (None, None, None, None)
                for repVersion, flavorList in d.iteritems():
                    for repFlavor in flavorList:
                        score = flavor.score(repFlavor)
                        if repVersion == version:
                            closeness = 100000
                        else:
                            closeness = version.closeness(repVersion)

                        if score is False: continue
                        if score < currentMatch[3]:
                            continue
                        elif score > currentMatch[3]:
                            currentMatch = (repVersion, repFlavor, closeness,
                                            score)
                        elif closeness < currentMatch[2]:
                            continue
                        else:
                            currentMatch = (repVersion, repFlavor, closeness,
                                            score)

                job = (name, (currentMatch[0], currentMatch[1]),
                              (version, flavor), currentMatch[0] is None)

            groupJobList.append((mark, job))

        # now iterate through groupJobList and update latestAvailable to
        # reflect the state of the mirror after this job completes
        for mark, job in groupJobList:
            name = job[0]
            oldVersion, oldFlavor = job[1]
            newVersion, newFlavor = job[2]

            d = latestAvailable.setdefault(name, {})

            if oldVersion in d and oldVersion.branch() == newVersion.branch():
                # If the old version is on the same branch as the new one,
                # replace the old with the new. If it's on a different
                # branch, we'll track both.
                d[oldVersion].remove(oldFlavor)
                if not d[oldVersion]: del d[oldVersion]

            flavorList = d.setdefault(newVersion, [])
            flavorList.append(newFlavor)

        jobList.append(groupJobList)

    return jobList

def mirrorSignatures(sourceRepos, targetRepos, currentMark, cfg,
                     test = False, syncSigs = False):
    log.debug("looking for new pgp keys")
    keyList = sourceRepos.getNewPGPKeys(cfg.host, currentMark)
    if test:
        log.debug("(not adding %d keys due to test mode)", len(keyList))
    else:
        log.debug("adding %d keys to target", len(keyList))
        targetRepos.addPGPKeyList(cfg.host, keyList)

    if syncSigs:
        log.debug("getting full trove list for signature sync")
        troveDict = sourceRepos.getTroveVersionList(cfg.host, { None : None })
        sigList = []
        for name, versionD in troveDict.iteritems():
            for version, flavorList in versionD.iteritems():
                for flavor in flavorList:
                    sigList.append((currentMark, (name, version, flavor)))
    else:
        log.debug("looking for new trove signatures")
        sigList = sourceRepos.getNewSigList(cfg.host, currentMark)
    # protection against duplicate items returned in the list by some servers
    sigList = list(set(sigList))
    sigList.sort(lambda a,b: cmp(a[0], b[0]))
    log.debug("%d new signatures are available" % len(sigList))

    # also weed out the signatures that don't belong on our label. Having none
    # left after this isn't that big a deal, so we  don't have to return
    if cfg.labels and len(sigList):
        sigList = [ x for x in sigList if
                            x[1][1].branch().label() in cfg.labels ]
        log.debug("after label filtering %d sigs are needed", len(sigList))

    log.debug("removing signatures for troves not yet mirrored")
    sigList = filterSigsWithoutTroves(targetRepos, currentMark, sigList)
    log.debug("%d signatures need to be mirrored", len(sigList))

    updateCount = 0
    if sigList:
        sigs = sourceRepos.getTroveSigs([ x[1] for x in sigList ])
        # build the ((n,v,f), signature) list only for the troves that have signatures
        sigs = [ (x[0][1], x[1]) for x in itertools.izip(sigList, sigs) if len(x[1]) > 0 ]
        if test:
            log.debug("not mirroring %d signatures due to test mode", len(sigs))
        else:
            log.debug("mirroring %d sigs", len(sigs))
            updateCount = targetRepos.setTroveSigs(sigs)

    return updateCount

def mirrorRepository(sourceRepos, targetRepos, cfg,
                     test = False, sync = False, syncSigs = False):
    # find the latest timestamp stored on the target mirror
    if sync:
        currentMark = -1
        targetRepos.setMirrorMark(cfg.host, currentMark)
    else:
        currentMark = targetRepos.getMirrorMark(cfg.host)
    log.debug("currently up to date through %d", int(currentMark))

    # first, mirror signatures for troves already mirrored
    updateCount = mirrorSignatures(sourceRepos, targetRepos, currentMark,
                                   cfg = cfg, test = test, syncSigs = syncSigs)

    # now find all of the troves we need from from the mirror source
{'fraction_non_alphanumeric': 0.07935272201523662, 'fraction_numerical': 0.00444864594339098, 'mean_word_length': 2.186392629340893, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2335520', 'n_tokens_mistral': 4973, 'n_tokens_neox': 4759, 'n_words': 1758}