<commit_before>#!/usr/bin/env node

'use strict';

// 3rd party modules

var chalk = require('chalk');
var fs = require('fs');
var npm = require('npm');
var path = require('path');
var when = require('when');

// Modules

var file = require('./src/file');
var npmCache = require('./src/npmCache');
var shrinkpack = require('./src/shrinkpack');
var shrinkwrap = require('./src/shrinkwrap');

// Run

loadNpm()
    .then(getConfig)
    .then(createDirectory)
    .then(getDeps)
    .then(prune)
    .then(getMissingDeps)
    .then(addMissingDeps)
    .then(updateShrinkwrap)
    .then(displaySummary, displayFailure);

// Implementation

function loadNpm() {
    return when.promise(function(resolve) {
        npm.load(resolve);
    });
}

function getConfig() {
    var projectPath = getPathToProject();
    var graphPath = path.join(projectPath, 'npm-shrinkwrap.json');
    var graph = require(graphPath);
    var shrinkpackPath = path.join(projectPath, 'node_shrinkwrap');

    return {
        deps: [],
        graph: graph,
        missingDeps: [],
        prunedPaths: [],
        path: {
            project: projectPath,
            graph: graphPath,
            shrinkpack: shrinkpackPath,
            npmCache: npm.config.get('cache')
        }
    };
}

function createDirectory(config) {
    if (!fs.existsSync(config.path.shrinkpack)) {
        fs.mkdirSync(config.path.shrinkpack);
    }
    return config;
}

function getDeps(config) {
    config.deps = shrinkwrap.toArray(config);
    return config;
}

function prune(config) {
    return shrinkpack.prune(config)
        .then(function(prunedPaths) {
            config.prunedPaths = prunedPaths;
            return config;
        }, function(err) {
            console.error(err);
            return err;
        }, function(msg) {
            console.info(chalk.red(msg));
            return msg;
        });
}

function getMissingDeps(config) {
    config.missingDeps = shrinkpack.getMissingPackages(config);
    return config;
}

function addMissingDeps(config) {
    return when.all(
        config.missingDeps.map(addMissingDep)
    ).then(function() {
        return config;
    });

    function addMissingDep(dep) {
        return when(dep)
            .then(addToNpmCache)
            .then(addToShrinkpack);
    }
}

function addToNpmCache(dep) {
    return npmCache.ensure(dep)
        .then(function() {
            return dep;
        }, function(err) {
            console.error(err);
            return err;
        }, function(msg) {
            return msg;
        });
}

function addToShrinkpack(dep) {
    return shrinkpack.addPackage(dep)
        .then(function() {
            return dep;
        }, function(err) {
            console.error(err);
            return err;
        }, function(msg) {
            console.info(chalk.green(msg));
            return msg;
        });
}

function updateShrinkwrap(config) {
    rewritePaths();

    return file.write(config.path.graph, JSON.stringify(config.graph, null, 2))
        .then(function() {
            return config;
        }, function(err) {
            return err;
        }, function(msg) {
            return msg;
        });

    function rewritePaths() {
        config.deps.forEach(rewritePath);
    }

    function rewritePath(dep) {
        dep.shrinkwrap.resolved = dep.tarball.shrinkpack.replace(config.path.project + '/', '');
    }
}

function getPathToProject() {
    return process.env.PWD || process.cwd();
}

function displaySummary(config) {
    console.info(
        'shrinkpack %s %s',
        chalk.green('+' + config.missingDeps.length),
        chalk.red('-' + config.prunedPaths.length)
    );
    process.exit(0);
}

function displayFailure(err) {
    console.error(
        chalk.red('Please raise an issue at %s\n\n%s'),
        chalk.underline('https://github.com/JamieMason/shrinkpack/issues'),
        String(err.stack).replace(/^/gm, '    ')
    );
    process.exit(1);
}
<commit_msg>fix(rewrite): update all resolved props inc. dupes<commit_after>#!/usr/bin/env node

'use strict';

// 3rd party modules

var chalk = require('chalk');
var fs = require('fs');
var npm = require('npm');
var path = require('path');
var when = require('when');

// Modules

var file = require('./src/file');
var npmCache = require('./src/npmCache');
var shrinkpack = require('./src/shrinkpack');
var shrinkwrap = require('./src/shrinkwrap');

// Run

loadNpm()
    .then(getConfig)
    .then(createDirectory)
    .then(getDeps)
    .then(prune)
    .then(getMissingDeps)
    .then(addMissingDeps)
    .then(updateShrinkwrap)
    .then(displaySummary, displayFailure);

// Implementation

function loadNpm() {
    return when.promise(function(resolve) {
        npm.load(resolve);
    });
}

function getConfig() {
    var projectPath = process.env.PWD || process.cwd();
    var graphPath = path.join(projectPath, 'npm-shrinkwrap.json');
    var graph = require(graphPath);
    var shrinkpackPath = path.join(projectPath, 'node_shrinkwrap');

    return {
        deps: [],
        graph: graph,
        missingDeps: [],
        prunedPaths: [],
        path: {
            project: projectPath,
            graph: graphPath,
            shrinkpack: shrinkpackPath,
            npmCache: npm.config.get('cache')
        }
    };
}

function createDirectory(config) {
    if (!fs.existsSync(config.path.shrinkpack)) {
        fs.mkdirSync(config.path.shrinkpack);
    }
    return config;
}

function getDeps(config) {
    config.deps = shrinkwrap.toArray(config);
    return config;
}

function prune(config) {
    return shrinkpack.prune(config)
        .then(function(prunedPaths) {
            config.prunedPaths = prunedPaths;
            return config;
        }, function(err) {
            console.error(err);
            return err;
        }, function(msg) {
            console.info(chalk.red(msg));
            return msg;
        });
}

function getMissingDeps(config) {
    config.missingDeps = shrinkpack.getMissingPackages(config);
    return config;
}

function addMissingDeps(config) {
    return when.all(
        config.missingDeps.map(addMissingDep)
    ).then(function() {
        return config;
    });

    function addMissingDep(dep) {
        return when(dep)
            .then(addToNpmCache)
            .then(addToShrinkpack);
    }

    function addToNpmCache(dep) {
        return npmCache.ensure(dep)
            .then(function() {
                return dep;
            }, function(err) {
                console.error(err);
                return err;
            }, function(msg) {
                return msg;
            });
    }

    function addToShrinkpack(dep) {
        return shrinkpack.addPackage(dep)
            .then(function() {
                return dep;
            }, function(err) {
                console.error(err);
                return err;
            }, function(msg) {
                console.info(chalk.green(msg));
                return msg;
            });
    }
}

function updateShrinkwrap(config) {

    rewritePaths();

    return file.write(config.path.graph, JSON.stringify(config.graph, null, 2))
        .then(function() {
            return config;
        }, function(err) {
            return err;
        }, function(msg) {
            return msg;
        });

    function rewritePaths() {
        shrinkwrap.recurse(config.graph, rewritePath);
    }

    function rewritePath(key, object) {
        object.resolved = 'node_shrinkwrap' + object.resolved.slice(object.resolved.lastIndexOf('/'), object.resolved.length);
    }
}

function displaySummary(config) {
    console.info(
        'shrinkpack %s %s',
        chalk.green('+' + config.missingDeps.length),
        chalk.red('-' + config.prunedPaths.length)
    );
    process.exit(0);
}

function displayFailure(err) {
    console.error(
        chalk.red('Please raise an issue at %s\n\n%s'),
        chalk.underline('https://github.com/JamieMason/shrinkpack/issues'),
        String(err.stack).replace(/^/gm, '    ')
    );
    process.exit(1);
}

{'fraction_non_alphanumeric': 0.1361770924014426, 'fraction_numerical': 0.0009949011317000372, 'mean_word_length': 2.227126805778491, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4449517', 'n_tokens_mistral': 2552, 'n_tokens_neox': 2443, 'n_words': 434}