<commit_before>import async from 'async'
import chalk from 'chalk'
import path from 'path'

import createGraph from '../graph/create'
import { layer as layerTopology } from '../graph/topology'
import taskPrune from '../tasks/prune'
import taskInstall from '../tasks/install'
import taskBridge from '../tasks/bridge'
import taskBuild from '../tasks/build'

import createReporter from '../reporter'

import { CONCURRENCY } from '../constants'

export default function(options) {
    let entryNode
    const reporter = createReporter()
    createGraph(path.resolve('.'), function(err, nodes) {
        if (err) end(err)

        entryNode = nodes[0]

        const layers = layerTopology(entryNode).reverse()

        async.series(
            [
                done => ensureHoist(done),
                done => pruneAndInstall(nodes, done),
                done => buildAndBridge(layers, done),
            ],
            end
        )
    })

    function ensureHoist(callback) {
        reporter.series('Hoisting...')
        taskInstall(entryNode, err => {
            reporter.clear()
            callback(err)
        })
    }

    function pruneAndInstall(nodes, callback) {
        reporter.series('Installing...')
        async.mapLimit(nodes, CONCURRENCY, pruneAndInstallNode, err => {
            reporter.clear()
            callback(err)
        })
    }

    function pruneAndInstallNode(node, callback) {
        const done = reporter.task(node.name)
        async.series(
            [done => taskPrune(node, done), done => taskInstall(node, done)],
            err => {
                done()
                callback(err)
            }
        )
    }

    function buildAndBridge(layers, callback) {
        async.mapSeries(layers, buildAndBridgeLayer, callback)
    }

    function buildAndBridgeLayer(layer, callback) {
        reporter.series(`Building Layer...`)
        async.mapLimit(layer, CONCURRENCY, buildAndBridgeNode, err => {
            reporter.clear()
            callback(err)
        })
    }

    function buildAndBridgeNode(node, callback) {
        const done = reporter.task(node.name)
        async.series(
            [
                done => taskBridge(node, done),
                done => taskBuild(node, entryNode, done),
            ],
            err => {
                done()
                callback(err)
            }
        )
    }
}

function end(err) {
    if (err) {
        console.log(err)
        console.log(chalk.red('Failure'))
        process.exit(1)
    } else {
        console.log(chalk.green('Success'))
        process.exit(0)
    }
}<commit_msg>Introduce the install-concurrency flag<commit_after>import async from 'async'
import chalk from 'chalk'
import path from 'path'
import invariant from 'invariant'

import createGraph from '../graph/create'
import { layer as layerTopology } from '../graph/topology'
import taskPrune from '../tasks/prune'
import taskInstall from '../tasks/install'
import taskBridge from '../tasks/bridge'
import taskBuild from '../tasks/build'

import createReporter from '../reporter'

import { CONCURRENCY } from '../constants'

export default function(options) {
    let entryNode
    const reporter = createReporter()
    createGraph(path.resolve('.'), function(err, nodes) {
        if (err) end(err)

        entryNode = nodes[0]

        const layers = layerTopology(entryNode).reverse()

        async.series(
            [
                installConcurrency(options) > 1
                    ? done => ensureHoist(done)
                    : null,
                done => pruneAndInstall(nodes, done),
                done => buildAndBridge(layers, done),
            ].filter(Boolean),
            end
        )
    })

    function ensureHoist(callback) {
        /*
         * TODO the ensure hoist should go on all the top level root nodes
         */
        reporter.series('Hoisting...')
        taskInstall(entryNode, err => {
            reporter.clear()
            callback(err)
        })
    }

    function pruneAndInstall(nodes, callback) {
        reporter.series(
            `Installing (max parallel ${installConcurrency(options)})...`
        )
        async.mapLimit(
            nodes,
            installConcurrency(options),
            pruneAndInstallNode,
            err => {
                reporter.clear()
                callback(err)
            }
        )
    }

    function pruneAndInstallNode(node, callback) {
        const done = reporter.task(node.name)
        async.series(
            [done => taskPrune(node, done), done => taskInstall(node, done)],
            err => {
                done()
                callback(err)
            }
        )
    }

    function buildAndBridge(layers, callback) {
        async.mapSeries(layers, buildAndBridgeLayer, callback)
    }

    function buildAndBridgeLayer(layer, callback) {
        reporter.series(`Building Layer...`)
        async.mapLimit(layer, CONCURRENCY, buildAndBridgeNode, err => {
            reporter.clear()
            callback(err)
        })
    }

    function buildAndBridgeNode(node, callback) {
        const done = reporter.task(node.name)
        async.series(
            [
                done => taskBridge(node, done),
                done => taskBuild(node, entryNode, done),
            ],
            err => {
                done()
                callback(err)
            }
        )
    }
}

function installConcurrency(options) {
    const c =
        typeof options.installConcurrency === 'number'
            ? options.installConcurrency
            : CONCURRENCY

    invariant(typeof c === 'number', 'Concurrency must be a number')

    return Math.max(c, 1)
}

function end(err) {
    if (err) {
        console.log(err)
        console.log(chalk.red('Failure'))
        process.exit(1)
    } else {
        console.log(chalk.green('Success'))
        process.exit(0)
    }
}
{'fraction_non_alphanumeric': 0.11483172670387282, 'fraction_numerical': 0.0013529511246406223, 'mean_word_length': 1.8269598470363289, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4301123', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1582, 'n_words': 353}