<commit_before>var nodeunit = require('nodeunit');
var assert = require('assert');
var karma = require('./build/karma/karma');
var mysql = require('mysql');
var database = require("./src/server/database.js");
var path = require("path");
var find = require("find");
var fs = require('fs-extra');
var rimraf = require("rimraf");
var childProcess = require("child_process");
var Q = require("q");
var request = require("request");
var os = require("os");
var nodeVersion = new(require("node-version").version)();
var util = require("util");
var beautify = require('js-beautify');


var config = require("./src/config.js");
var vagrant = require("./src/vagrant.js");

var copyModifiedJson = require("cauldron").copyModifiedJson;
var gitUtil = require("cauldron").gitUtil;
var spawnProcess = require("cauldron").spawnProcess;

var jadePreprocessor = require("./build/karma/jadePreprocessor.js");

var allTests = "**/*.test.js";
var slowTests = "**/*.slow.test.js";
var smokeTests = "**/*.smoke.test.js";
var clientCode = "src/client/**";

task = require("./build/jake-util.js").extendTask(task, jake);

task("default", ["verifyNodeVersion", "lint", "writeSampleConfig", "test"], function() {});

task("verifyClientCode", ["lint", "testClient"], function() {});


desc("lint");
task("lint", function() {

    var list = listNonImportedFiles();
    list.include("**/*.js");
    list.exclude("jadeRuntime.js");

    var runner = require("./build/lint/lint_runner.js");

    var a = runner.validateFileList(list, { 
        evil: true  // allowing eval, only used in test code so far
    });
    assert.ok(a, "lint failed");
});

desc("Formats javascript files.");
task("beautify", ["verifyEmptyGitStatus"], function() {

    var list = listNonImportedFiles();
    list.include("**/*.js");

    var readFile = Q.nbind(fs.readFile);
    var writeFile = Q.nbind(fs.writeFile);

    var promises = list.toArray().map(function(filename) {
        return readFile(filename, "utf8")
            .then(function(contents) {
                return writeFile(filename, beautify(contents, {
                            indent_size: 4
                        }));
            });
    });

    return Q.all(promises);
});

desc("write sample config file");
task("writeSampleConfig", function() {

    var defaults = config.getDefaults();

    fs.writeFileSync("./sample.config.json", JSON.stringify(defaults, null, "    "));
});

desc("test everything");
task("test", ["testClient", "testSmoke", "testRemaining", "testSlow"]);

task("testClient", function() {

    return karma.runTests(path.resolve("./build/karma/karma.conf.js"));
});

function runTestsWithNodeunit(testList) {
    var reporter = nodeunit.reporters["default"];
    reporter.run(testList, null, function(failures) {
        assert.ifError(failures);
        complete();
    });
}

task("commonTestPrequisites", ["lint", "prepareTestDatabase", "buildClientBundle"], function() {});

task("testSlow", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(slowTests);
    testList.exclude(clientCode);

    runTestsWithNodeunit(testList.toArray());
}, {
    async: true
});

task("testSmoke", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(smokeTests);
    testList.exclude(clientCode);

    runTestsWithNodeunit(testList.toArray());
}, {
    async: true
});

task("testRemaining", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(allTests);

    testList.exclude(clientCode);
    testList.exclude(slowTests);
    testList.exclude(smokeTests);

    runTestsWithNodeunit(testList.toArray());
}, {
    async: true
});

task("createTestDatabase", function() {

    database.ensureTestDatabaseIsClean(function(err) {
        assert.ifError(err);
        complete();
    });
}, {
    async: true
});

task("prepareTestDatabase", ["createTestDatabase", "runMigrations"]);


function assertExecFileSucceeded(execFileResults) {

    var stdout = execFileResults[0];
    var stderr = execFileResults[1];

    console.log("stdout:\n" + stdout.toString());

    if (stderr.trim().length > 0) {
        throw new Error("Have error output: " + stderr.toString());
    }
}
<commit_msg>refactor jakefile.js's runTestsWithNodeunit<commit_after>var nodeunit = require('nodeunit');
var assert = require('assert');
var karma = require('./build/karma/karma');
var mysql = require('mysql');
var database = require("./src/server/database.js");
var path = require("path");
var find = require("find");
var fs = require('fs-extra');
var rimraf = require("rimraf");
var childProcess = require("child_process");
var Q = require("q");
var request = require("request");
var os = require("os");
var nodeVersion = new(require("node-version").version)();
var util = require("util");
var beautify = require('js-beautify');


var config = require("./src/config.js");
var vagrant = require("./src/vagrant.js");

var copyModifiedJson = require("cauldron").copyModifiedJson;
var gitUtil = require("cauldron").gitUtil;
var spawnProcess = require("cauldron").spawnProcess;

var jadePreprocessor = require("./build/karma/jadePreprocessor.js");

var allTests = "**/*.test.js";
var slowTests = "**/*.slow.test.js";
var smokeTests = "**/*.smoke.test.js";
var clientCode = "src/client/**";

task = require("./build/jake-util.js").extendTask(task, jake);

task("default", ["verifyNodeVersion", "lint", "writeSampleConfig", "test"], function() {});

task("verifyClientCode", ["lint", "testClient"], function() {});


desc("lint");
task("lint", function() {

    var list = listNonImportedFiles();
    list.include("**/*.js");
    list.exclude("jadeRuntime.js");

    var runner = require("./build/lint/lint_runner.js");

    var a = runner.validateFileList(list, { 
        evil: true  // allowing eval, only used in test code so far
    });
    assert.ok(a, "lint failed");
});

desc("Formats javascript files.");
task("beautify", ["verifyEmptyGitStatus"], function() {

    var list = listNonImportedFiles();
    list.include("**/*.js");

    var readFile = Q.nbind(fs.readFile);
    var writeFile = Q.nbind(fs.writeFile);

    var promises = list.toArray().map(function(filename) {
        return readFile(filename, "utf8")
            .then(function(contents) {
                return writeFile(filename, beautify(contents, {
                            indent_size: 4
                        }));
            });
    });

    return Q.all(promises);
});

desc("write sample config file");
task("writeSampleConfig", function() {

    var defaults = config.getDefaults();

    fs.writeFileSync("./sample.config.json", JSON.stringify(defaults, null, "    "));
});

desc("test everything");
task("test", ["testClient", "testSmoke", "testRemaining", "testSlow"]);

task("testClient", function() {

    return karma.runTests(path.resolve("./build/karma/karma.conf.js"));
});

function runTestsWithNodeunit(testList) {

    return Q()
    .then(function() {

        var deferred = Q.defer();

        var reporter = nodeunit.reporters["default"];
        reporter.run(testList, null, function(failures) {
            if (failures) {
                deferred.reject(failures);
            } else {
                deferred.resolve();
            }
        });

        return deferred.promise;
    });
}

task("commonTestPrequisites", ["lint", "prepareTestDatabase", "buildClientBundle"], function() {});

task("testSlow", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(slowTests);
    testList.exclude(clientCode);

    return runTestsWithNodeunit(testList.toArray());
});

task("testSmoke", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(smokeTests);
    testList.exclude(clientCode);

    return runTestsWithNodeunit(testList.toArray());
});

task("testRemaining", ["commonTestPrequisites"], function() {

    var testList = listNonImportedFiles();

    testList.include(allTests);

    testList.exclude(clientCode);
    testList.exclude(slowTests);
    testList.exclude(smokeTests);

    return runTestsWithNodeunit(testList.toArray());
});

task("createTestDatabase", function() {

    database.ensureTestDatabaseIsClean(function(err) {
        assert.ifError(err);
        complete();
    });
}, {
    async: true
});

task("prepareTestDatabase", ["createTestDatabase", "runMigrations"]);

{'fraction_non_alphanumeric': 0.17567248702218027, 'fraction_numerical': 0.0008258612553091081, 'mean_word_length': 3.977686435701703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45176', 'n_tokens_mistral': 2645, 'n_tokens_neox': 2539, 'n_words': 493}