<commit_before>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
"""
Cache of changesets.
"""
from StringIO import StringIO
import errno
import os
import itertools
import tempfile

from conary import trove

from conary.deps import deps
from conary.lib import sha1helper
from conary.lib import util
from conary.repository import changeset
from conary.repository import datastore
from conary.repository import errors
from conary.repository import filecontents
from conary.repository import trovesource

class CachingTroveSource:
    def __init__(self, troveSource, cacheDir, readOnly=False, depsOnly=False):
        self._troveSource = troveSource
        util.mkdirChain(cacheDir)
        self._depsOnly = depsOnly
        self._cache = RepositoryCache(cacheDir, readOnly=readOnly)

    def __getattr__(self, key):
        return getattr(self._troveSource, key)

    def getTroves(self, troveList, withFiles=True, callback = None):
        if self._depsOnly:
            return self._troveSource.getTroves(troveList, withFiles=withFiles,
                                               callback=callback)
        return self._cache.getTroves(self._troveSource, troveList,
                                     withFiles=withFiles, callback = None)

    def hasTroves(self, troveList):
        if self._depsOnly:
            return self._troveSource.hasTroves(troveList)
        return self._cache.hasTroves(self._troveSource, troveList)

    def hasTrove(self, name, version, flavor):
        return self.hasTroves([(name, version, flavor)])[name, version, flavor]

    def getTrove(self, name, version, flavor, withFiles=True, callback=None):
        trv = self.getTroves([(name, version, flavor)], withFiles=withFiles,
                              callback=callback)[0]
        if trv is None:
            raise errors.TroveMissing(name, version)
        return trv

    def resolveDependenciesByGroups(self, groupTroves, depList):
        if not (groupTroves and depList):
            return {}
        return self._cache.resolveDependenciesByGroups(self._troveSource, 
                                                       groupTroves,
                                                       depList)

    def getFileContents(self, fileList, callback = None):
        if self._depsOnly:
            return self._troveSource.getFileContents(fileList,
                                                     callback=callback)
        return self._cache.getFileContents(self._troveSource,
                                           fileList, callback=callback)

class DependencyResultList(trove.TroveTupleList):
    def get(self):
        rv = [ (x.name(), x.version().copy(), x.flavor()) for x in self.iter() ]
        [ x[1].resetTimeStamps() for x in rv ]
        return rv


class RepositoryCache(object):
    """
        We cache changeset files by component.  When conary is fixed, we'll
        be able to combine the download of these troves.
    """
    def __init__(self, cacheDir, readOnly=False, depsOnly=False):
        self.root = cacheDir
        self.store = DataStore(cacheDir)
        self.readOnly = readOnly
        self.depsOnly = depsOnly
        self.fileCache = LazyFileCache(300)

    def hashGroupDeps(self, groupTroves, depClass, dependency):
        depSet = deps.DependencySet()
        depSet.addDep(depClass, dependency)
        frz = depSet.freeze()
        troveList = sorted(self.hashTrove(withFiles=False,
                                          withFileContents=False,
                                          *x.getNameVersionFlavor())
                           for x in groupTroves)
        str = '[1]%s%s%s' % (len(frz), frz, ''.join(troveList))
        return sha1helper.sha1ToString(sha1helper.sha1String(str))

    def hashFile(self, fileId, fileVersion):
        # we add extra delimiters here because we can be sure they they
        # will result in a unique string for each n,v,f
        return sha1helper.sha1ToString(
                    sha1helper.sha1String('[0]%s=%s' % (fileId, fileVersion)))

    def hashTrove(self, name, version, flavor, withFiles, withFileContents):
        # we add extra delimiters here because we can be sure they they
        # will result in a unique string for each n,v,f
        return sha1helper.sha1ToString(
                sha1helper.sha1String('%s=%s[%s]%s%s' % (name, version, flavor, withFiles, withFileContents)))

    def getChangeSetsForTroves(self, repos, troveList, withFiles=True,
                               withFileContents=True, callback=None):
        return self.getChangeSets(repos,
                                  [(x[0], (None, None), (x[1], x[2]), False) 
                                   for x in troveList ],
                                   withFiles, withFileContents,
                                   callback=callback)


    def getTroves(self, repos, troveList, withFiles=True,
                  withFileContents=False, callback=None):
        csList = self.getChangeSetsForTroves(repos, troveList, withFiles,
                                             withFileContents, callback)
        l = []
        for cs, info in itertools.izip(csList, troveList):
            if cs is None:
                l.append(None)
                continue
            troveCs = cs.getNewTroveVersion(*info)
            # trove integrity checks don't work when file information is
            # excluded
            t = trove.Trove(troveCs, skipIntegrityChecks = not withFiles)
            l.append(t)
        return l

    def resolveDependenciesByGroups(self, repos, groupTroves, depList):
        allToFind = []
        allFound = []
        allMissing = []
        for depSet in depList:
            d = {}
            toFind = deps.DependencySet()
            found  = []
            missingIdx = []
            allToFind.append(toFind)
            allFound.append(found)
            allMissing.append(missingIdx)
            for idx, (depClass, dependency) in enumerate(depSet.iterDeps(sort=True)):
                depHash = str(self.hashGroupDeps(groupTroves, depClass, <commit_msg>Avoid having tons of open files all the time<commit_after>#
# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
#
"""
Cache of changesets.
"""
from StringIO import StringIO
import errno
import os
import itertools
import tempfile

from conary import trove

from conary.deps import deps
from conary.lib import sha1helper
from conary.lib import util
from conary.repository import changeset
from conary.repository import datastore
from conary.repository import errors
from conary.repository import filecontents
from conary.repository import trovesource

class CachingTroveSource:
    def __init__(self, troveSource, cacheDir, readOnly=False, depsOnly=False):
        self._troveSource = troveSource
        util.mkdirChain(cacheDir)
        self._depsOnly = depsOnly
        self._cache = RepositoryCache(cacheDir, readOnly=readOnly)

    def __getattr__(self, key):
        return getattr(self._troveSource, key)

    def getTroves(self, troveList, withFiles=True, callback = None):
        if self._depsOnly:
            return self._troveSource.getTroves(troveList, withFiles=withFiles,
                                               callback=callback)
        return self._cache.getTroves(self._troveSource, troveList,
                                     withFiles=withFiles, callback = None)

    def hasTroves(self, troveList):
        if self._depsOnly:
            return self._troveSource.hasTroves(troveList)
        return self._cache.hasTroves(self._troveSource, troveList)

    def hasTrove(self, name, version, flavor):
        return self.hasTroves([(name, version, flavor)])[name, version, flavor]

    def getTrove(self, name, version, flavor, withFiles=True, callback=None):
        trv = self.getTroves([(name, version, flavor)], withFiles=withFiles,
                              callback=callback)[0]
        if trv is None:
            raise errors.TroveMissing(name, version)
        return trv

    def resolveDependenciesByGroups(self, groupTroves, depList):
        if not (groupTroves and depList):
            return {}
        return self._cache.resolveDependenciesByGroups(self._troveSource, 
                                                       groupTroves,
                                                       depList)

    def getFileContents(self, fileList, callback = None):
        if self._depsOnly:
            return self._troveSource.getFileContents(fileList,
                                                     callback=callback)
        return self._cache.getFileContents(self._troveSource,
                                           fileList, callback=callback)

class DependencyResultList(trove.TroveTupleList):
    def get(self):
        rv = [ (x.name(), x.version().copy(), x.flavor()) for x in self.iter() ]
        [ x[1].resetTimeStamps() for x in rv ]
        return rv


class RepositoryCache(object):
    """
        We cache changeset files by component.  When conary is fixed, we'll
        be able to combine the download of these troves.
    """
    def __init__(self, cacheDir, readOnly=False, depsOnly=False):
        self.root = cacheDir
        self.store = DataStore(cacheDir)
        self.readOnly = readOnly
        self.depsOnly = depsOnly
        self.fileCache = LazyFileCache(300)

    def hashGroupDeps(self, groupTroves, depClass, dependency):
        depSet = deps.DependencySet()
        depSet.addDep(depClass, dependency)
        frz = depSet.freeze()
        troveList = sorted(self.hashTrove(withFiles=False,
                                          withFileContents=False,
                                          *x.getNameVersionFlavor())
                           for x in groupTroves)
        str = '[1]%s%s%s' % (len(frz), frz, ''.join(troveList))
        return sha1helper.sha1ToString(sha1helper.sha1String(str))

    def hashFile(self, fileId, fileVersion):
        # we add extra delimiters here because we can be sure they they
        # will result in a unique string for each n,v,f
        return sha1helper.sha1ToString(
                    sha1helper.sha1String('[0]%s=%s' % (fileId, fileVersion)))

    def hashTrove(self, name, version, flavor, withFiles, withFileContents):
        # we add extra delimiters here because we can be sure they they
        # will result in a unique string for each n,v,f
        return sha1helper.sha1ToString(
                sha1helper.sha1String('%s=%s[%s]%s%s' % (name, version, flavor, withFiles, withFileContents)))

    def getChangeSetsForTroves(self, repos, troveList, withFiles=True,
                               withFileContents=True, callback=None):
        return self.getChangeSets(repos,
                                  [(x[0], (None, None), (x[1], x[2]), False) 
                                   for x in troveList ],
                                   withFiles, withFileContents,
                                   callback=callback)


    def getTroves(self, repos, troveList, withFiles=True,
                  withFileContents=False, callback=None):
        csList = self.getChangeSetsForTroves(repos, troveList, withFiles,
                                             withFileContents, callback)
        l = []
        for cs, info in itertools.izip(csList, troveList):
            if cs is None:
                l.append(None)
                continue
            troveCs = cs.getNewTroveVersion(*info)
            # trove integrity checks don't work when file information is
            # excluded
            t = trove.Trove(troveCs, skipIntegrityChecks = not withFiles)
            l.append(t)
        self.fileCache.close()
        return l

    def resolveDependenciesByGroups(self, repos, groupTroves, depList):
        allToFind = []
        allFound = []
        allMissing = []
        for depSet in depList:
            d = {}
            toFind = deps.DependencySet()
            found  = []
            missingIdx = []
            allToFind.append(toFind)
            allFound.append(found)
            allMissing.append(missingIdx)
            for idx, (depClass, dependency) in enumerate(depSet.iterDeps(sort=True)):
                depHash = str(self.hashGroupDeps(groupTroves, depClass, 
{'fraction_non_alphanumeric': 0.0855327468230694, 'fraction_numerical': 0.005050505050505051, 'mean_word_length': 2.037357743691242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4001119', 'n_tokens_mistral': 3454, 'n_tokens_neox': 3301, 'n_words': 841}