<commit_before>var URL = require('url'),
    crypto = require('crypto'),
    seq = require('seq'),
    error = require('../error'),
    query = require('../query');

// Helper function for determining the order in which the hashes can be computed and the assets
// renamed. The challenge lies in the fact that updating a relation to point at <hash>.<extension>
// will change the hash of the asset that owns the relation.
// Needless to say this will fail if the graph of assets to be renamed has cycles, so be careful.
function findAssetRenameOrderBatches(assetGraph, queryObj) {
    var batches = [],
        clone = assetGraph.clone(),
        assetMatcher = query.queryObjToMatcherFunction(queryObj),
        currentBatch;

    // Remove all assets not matched by queryObj
    clone.findAssets().forEach(function (asset) { // FIXME: Query
        if (!assetMatcher(asset)) {
            clone.removeAsset(asset, true); // cascade (also remove incoming and outgoing relations)
        }
    });

    while (clone.assets.length) {
        currentBatch = [];
        clone.findAssets().forEach(function (asset) {
            if (!clone.findRelations({from: asset}).length) {
                currentBatch.push(asset);
            }
        });

        currentBatch.forEach(function (asset) {
            clone.removeAsset(asset, true); // Cascade
        });

        if (currentBatch.length === 0) {
            throw new Error("transforms.renameAssetsToMD5Prefix: Couldn't find a suitable rename order due to cycles in the selection");
        }
        batches.push(currentBatch);
    }
    return batches;
};

module.exports = function (queryObj) {
    return function renameAssetsToMD5Prefix(err, assetGraph, cb) {
        if (err) {
            throw err;
        }
        function renameBatch(assetBatch, cb) {
            seq.ap(assetBatch).
                parMap(function (asset) {<commit_msg>transforms.renameAssetsToMD5Prefix: Set a concurrency limit of 40.<commit_after>var URL = require('url'),
    crypto = require('crypto'),
    seq = require('seq'),
    error = require('../error'),
    query = require('../query');

// Helper function for determining the order in which the hashes can be computed and the assets
// renamed. The challenge lies in the fact that updating a relation to point at <hash>.<extension>
// will change the hash of the asset that owns the relation.
// Needless to say this will fail if the graph of assets to be renamed has cycles, so be careful.
function findAssetRenameOrderBatches(assetGraph, queryObj) {
    var batches = [],
        clone = assetGraph.clone(),
        assetMatcher = query.queryObjToMatcherFunction(queryObj),
        currentBatch;

    // Remove all assets not matched by queryObj
    clone.findAssets().forEach(function (asset) { // FIXME: Query
        if (!assetMatcher(asset)) {
            clone.removeAsset(asset, true); // cascade (also remove incoming and outgoing relations)
        }
    });

    while (clone.assets.length) {
        currentBatch = [];
        clone.findAssets().forEach(function (asset) {
            if (!clone.findRelations({from: asset}).length) {
                currentBatch.push(asset);
            }
        });

        currentBatch.forEach(function (asset) {
            clone.removeAsset(asset, true); // Cascade
        });

        if (currentBatch.length === 0) {
            throw new Error("transforms.renameAssetsToMD5Prefix: Couldn't find a suitable rename order due to cycles in the selection");
        }
        batches.push(currentBatch);
    }
    return batches;
};

module.exports = function (queryObj) {
    return function renameAssetsToMD5Prefix(err, assetGraph, cb) {
        if (err) {
            throw err;
        }
        function renameBatch(assetBatch, cb) {
            seq.ap(assetBatch).
                parMap(40, function (asset) {
{'fraction_non_alphanumeric': 0.1091285234031549, 'fraction_numerical': 0.0028445823635893457, 'mean_word_length': 2.5291970802919708, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7379367', 'n_tokens_mistral': 1035, 'n_tokens_neox': 1000, 'n_words': 321}