<commit_before>#
# Copyright (c) 2007 rPath, Inc.
#
# All Rights Reserved
#

from conary.conaryclient import ConaryClient
from conary import trove, files
from conary import conarycfg
from conary.lib import util
import conary

import os

def sortMountPoints(mounts):
    """Sorts mount points by specificity by counting os.path.sep instances."""
    mounts = [util.normpath(x) for x in mounts]

    # sort mounts by specificity, more specific mountpoints first.
    mounts = sorted(mounts, key = lambda x: x.count(os.path.sep), reverse = True)

    # / is special, put it at the end
    if os.path.sep in mounts:
        mounts.remove(os.path.sep)
        mounts.append(os.path.sep)

    return mounts


def calculatePartitionSizes(cs, mounts):
    """Iterate over every file in a changeset and returns a sum of files inside
       each directory specified by "mounts"."""
    mounts = sortMountPoints(mounts)
    mountDict = dict.fromkeys(mounts, 0)

    seen = {}
    q = util.IterableQueue()
    for n, v, f in cs.getPrimaryTroveList():
        q.add((n, v, f))
    for n, v, f in q:
        trv = trove.Trove(cs.getNewTroveVersion(n, v, f))

        for (name, version, flavor), byDefault, strongRef in trv.iterTroveListInfo():
            if not byDefault:
                seen[(name, version, flavor)] = True
                continue
            q.add((name, version, flavor))

        if (n, v, f) in seen:
            continue

        for pathId, path, fileId, fVer in trv.iterFileList():
            fStr = cs.getFileChange(None, fileId)
            fObj = files.frozenFileContentInfo(fStr)

            if type(fObj) == files.RegularFileStream:
                for mount in mounts:
                    if path.startswith(mount):
                        mountDict[mount] += fObj.size()
                        break
        seen[(n, v, f)] = True
    return mountDict, sum(mountDict.values())<commit_msg>Fix major bug in filesystem size calculation. (RBL-2330)<commit_after>#
# Copyright (c) 2007 rPath, Inc.
#
# All Rights Reserved
#

from conary.conaryclient import ConaryClient
from conary import trove, files
from conary import conarycfg
from conary.lib import util
import conary

import os

def sortMountPoints(mounts):
    """Sorts mount points by specificity by counting os.path.sep instances."""
    mounts = [util.normpath(x) for x in mounts]

    # sort mounts by specificity, more specific mountpoints first.
    mounts = sorted(mounts, key = lambda x: x.count(os.path.sep), reverse = True)

    # / is special, put it at the end
    if os.path.sep in mounts:
        mounts.remove(os.path.sep)
        mounts.append(os.path.sep)

    return mounts


def calculatePartitionSizes(cs, mounts):
    """Iterate over every file in a changeset and returns a sum of files inside
       each directory specified by "mounts"."""
    mounts = sortMountPoints(mounts)
    mountDict = dict.fromkeys(mounts, 0)

    seen = {}
    q = util.IterableQueue()
    for n, v, f in cs.getPrimaryTroveList():
        q.add((n, v, f))
    for n, v, f in q:
        trv = trove.Trove(cs.getNewTroveVersion(n, v, f))

        for (name, version, flavor), byDefault, strongRef in trv.iterTroveListInfo():
            if not byDefault:
                continue
            q.add((name, version, flavor))

        if (n, v, f) in seen:
            continue

        for pathId, path, fileId, fVer in trv.iterFileList():
            fStr = cs.getFileChange(None, fileId)
            fObj = files.frozenFileContentInfo(fStr)

            if type(fObj) == files.RegularFileStream:
                for mount in mounts:
                    if path.startswith(mount):
                        blockSize = 4096
                        realSize = fObj.size()
                        nearestBlock = (realSize / blockSize + 1) * blockSize
                        mountDict[mount] += nearestBlock
                        break
        seen[(n, v, f)] = True
    return mountDict, sum(mountDict.values())
{'fraction_non_alphanumeric': 0.09917147878483555, 'fraction_numerical': 0.004770273663068039, 'mean_word_length': 2.2522448979591836, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2063438', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1171, 'n_words': 379}