<commit_before>var sys = require('sys')
    exec = require('child_process').exec,
    spawn = require('child_process').spawn;

var NO_OP = function() {};

desc('Build the application.');
task('default', ['test:all'], NO_OP);

namespace('test', function() {
    require.paths.unshift(__dirname + '/deps/nodeunit/lib');
    var testrunner = require('nodeunit').reporters.default;

    desc('Runs all unit tests');
    task('unit', [], function() { testrunner.run(['test/unit']); });

    desc('Runs all functional tests');
    task('functional', [], function() {
        /*
        var app = spawn('node', ['app.js'], {cwd: __dirname, env: process.env, customFds: [-1, -1, -1]});
        app.stdout.setEncoding('utf8');
        app.stdout.on('data', function (data) {
            if (data.indexOf('Server running') === 0) {
            */
                testrunner.run(['test/functional']);
                /*
                app.kill();
            }
        });
        */
    });

    desc('Runs all tests');
    task('all', ['test:unit', 'test:functional'], NO_OP);
});

namespace('package', function() {
    function packagePath(packageName) {
        return '/usr/local/lib/node/.npm/' + packageName + '/active/package';
    }

    function localPath(packageName) {
        return __dirname + '/deps/' + packageName;
    }

    desc("Copies an npm package to the lib directory, so the application doesn't depend on preinstalled packages")
    task('unpack', [], function() {
        exec('[ -d deps ] || mkdir deps ]', NO_OP);
        for (var i = 0; i < arguments.length; i++) {
            var command = 'cp -RL ' + packagePath(arguments[i]) + ' ' + localPath(arguments[i]);
            exec(command, function(error, stdout, stderr) {
                var message = (error == null)
                    ? packagePath(arguments[i]) + ' => ' + localPath(arguments[i])
                    : error + '\n' + stderr;
                console.log(message);
            });
        }
    });
});<commit_msg>Cleaned up the unpack code<commit_after>var sys = require('sys')
    exec = require('child_process').exec,
    spawn = require('child_process').spawn;

var NO_OP = function() {};

desc('Build the application.');
task('default', ['test:all'], NO_OP);

namespace('test', function() {
    require.paths.unshift(__dirname + '/deps/nodeunit/lib');
    var testrunner = require('nodeunit').reporters.default;

    desc('Runs all unit tests');
    task('unit', [], function() { testrunner.run(['test/unit']); });

    desc('Runs all functional tests');
    task('functional', [], function() {
        /*
        var app = spawn('node', ['app.js'], {cwd: __dirname, env: process.env, customFds: [-1, -1, -1]});
        app.stdout.setEncoding('utf8');
        app.stdout.on('data', function (data) {
            if (data.indexOf('Server running') === 0) {
            */
                testrunner.run(['test/functional']);
                /*
                app.kill();
            }
        });
        */
    });

    desc('Runs all tests');
    task('all', ['test:unit', 'test:functional'], NO_OP);
});

namespace('package', function() {
    var DEPENDENCIES_DIR = __dirname + '/deps';

    function packagePath(packageName) {
        return '/usr/local/lib/node/.npm/' + packageName + '/active/package';
    }

    function localPath(packageName) {
        return DEPENDENCIES_DIR + '/' + packageName;
    }

    desc("Copies an npm package to the lib directory, so the application doesn't depend on preinstalled packages")
    task('unpack', [], function() {
        var mkdir = '[ -d ' + DEPENDENCIES_DIR + ' ] || mkdir ' + DEPENDENCIES_DIR;
        exec(mkdir, NO_OP);
        for (var i = 0; i < arguments.length; i++) {
            var copy = 'cp -RL ' + packagePath(arguments[i]) + ' ' + localPath(arguments[i]);
            exec(copy, function(error, stdout, stderr) {
                var message = (error == null)
                    ? packagePath(arguments[i]) + ' => ' + localPath(arguments[i])
                    : error + '\n' + stderr;
                console.log(message);
            });
        }
    });
});
{'fraction_non_alphanumeric': 0.17870999030067894, 'fraction_numerical': 0.002909796314258002, 'mean_word_length': 2.3454987834549876, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5340205', 'n_tokens_mistral': 1247, 'n_tokens_neox': 1207, 'n_words': 269}