<commit_before>var assert = require('assert'),
    fs = require('fs'),
    glob = require('glob'),
    lumbar = require('../../lib/lumbar'),
    path = require('path'),
    wrench = require('wrench');

var counter = 0;
try { fs.mkdirSync('/tmp/lumbar-test', 0755); } catch (err) {}

exports.testDir = function(testName, configFile) {
  var outdir = '/tmp/lumbar-test/' + testName + '-' + path.basename(configFile) + '-' + Date.now() + '-' + (counter++);
  fs.mkdirSync(outdir, 0755);
  return outdir;
};
exports.assertExpected = function(outdir, expectedDir, configFile) {
  var expectedFiles = glob.globSync(expectedDir + '/**/*.*').map(function(fileName) {
        return fileName.substring(expectedDir.length);
      }).filter(function(file) { return !/\/$/.test(file); }).sort(),
      generatedFiles = glob.globSync(outdir + '/**/*.*').map(function(fileName) {
        return fileName.substring(outdir.length);
      }).filter(function(file) { return !/\/$/.test(file); }).sort();
  assert.deepEqual(generatedFiles, expectedFiles, configFile + ': file list matches' + JSON.stringify(expectedFiles) + JSON.stringify(generatedFiles));

  generatedFiles.forEach(function(fileName) {
    var generatedContent = fs.readFileSync(outdir + fileName, 'utf8'),
        expectedContent = fs.readFileSync(expectedDir + fileName, 'utf8');
    assert.equal(generatedContent, expectedContent, configFile + ':' + fileName + ': content matches');
  });
};

exports.runTest = function(configFile, expectedDir, options) {
  return function(done) {
    var outdir = exports.testDir('lumbar', configFile);
    this.title += ' ' + outdir;

    options = options || {};
    options.outdir = outdir;

    var expectedFiles = glob.globSync(expectedDir + '/**/*.{css,js}').map(function(fileName) {
          return fileName.substring(expectedDir.length);
        }).sort(),
        seenFiles = [];

    var arise = lumbar.init(configFile, options || {outdir: outdir});
    arise.on('output', function(status) {
      var statusFile = status.fileName.substring(outdir.length);
      if (!expectedFiles.some(function(fileName) { return statusFile === fileName; })) {
        assert.fail(undefined, status.fileName, configFile + ':' + statusFile + ': missing from expected list');
      } else {
        seenFiles.push(statusFile);
      }
      if (seenFiles.length < expectedFiles.length) {
        return;
      }

      exports.assertExpected(outdir, expectedDir, configFile);

      seenFiles = seenFiles.sort();
      assert.deepEqual(seenFiles, expectedFiles, 'seen file list matches');

      // Cleanup (Do cleanup here so the files remain for the failure case)
      wrench.rmdirSyncRecursive(outdir);

      done();
    });
    arise.build(undefined, function(err) {
      throw err;
    });
  };
}
<commit_msg>Add lumbar test build callback expectation<commit_after>var assert = require('assert'),
    fs = require('fs'),
    glob = require('glob'),
    lumbar = require('../../lib/lumbar'),
    path = require('path'),
    wrench = require('wrench');

var counter = 0;
try { fs.mkdirSync('/tmp/lumbar-test', 0755); } catch (err) {}

exports.testDir = function(testName, configFile) {
  var outdir = '/tmp/lumbar-test/' + testName + '-' + path.basename(configFile) + '-' + Date.now() + '-' + (counter++);
  fs.mkdirSync(outdir, 0755);
  return outdir;
};
exports.assertExpected = function(outdir, expectedDir, configFile) {
  var expectedFiles = glob.globSync(expectedDir + '/**/*.*').map(function(fileName) {
        return fileName.substring(expectedDir.length);
      }).filter(function(file) { return !/\/$/.test(file); }).sort(),
      generatedFiles = glob.globSync(outdir + '/**/*.*').map(function(fileName) {
        return fileName.substring(outdir.length);
      }).filter(function(file) { return !/\/$/.test(file); }).sort();
  assert.deepEqual(generatedFiles, expectedFiles, configFile + ': file list matches' + JSON.stringify(expectedFiles) + JSON.stringify(generatedFiles));

  generatedFiles.forEach(function(fileName) {
    var generatedContent = fs.readFileSync(outdir + fileName, 'utf8'),
        expectedContent = fs.readFileSync(expectedDir + fileName, 'utf8');
    assert.equal(generatedContent, expectedContent, configFile + ':' + fileName + ': content matches');
  });
};

exports.runTest = function(configFile, expectedDir, options) {
  return function(done) {
    var outdir = exports.testDir('lumbar', configFile);
    this.title += ' ' + outdir;

    options = options || {};
    options.outdir = outdir;

    var expectedFiles = glob.globSync(expectedDir + '/**/*.{css,js}').map(function(fileName) {
          return fileName.substring(expectedDir.length);
        }).sort(),
        seenFiles = [];

    var arise = lumbar.init(configFile, options || {outdir: outdir});
    arise.on('output', function(status) {
      var statusFile = status.fileName.substring(outdir.length);
      if (!expectedFiles.some(function(fileName) { return statusFile === fileName; })) {
        assert.fail(undefined, status.fileName, configFile + ':' + statusFile + ': missing from expected list');
      } else {
        seenFiles.push(statusFile);
      }
      if (seenFiles.length < expectedFiles.length) {
        return;
      }

      exports.assertExpected(outdir, expectedDir, configFile);

      seenFiles = seenFiles.sort();
      assert.deepEqual(seenFiles, expectedFiles, 'seen file list matches');

      // Cleanup (Do cleanup here so the files remain for the failure case)
      wrench.rmdirSyncRecursive(outdir);

      done();
    });
    var retCount = 0;
    arise.build(undefined, function(err) {
      retCount++;
      if (retCount > 1) {
        throw new Error('Build callback executed multiple times');
      }

      throw err;
    });
  };
}

{'fraction_non_alphanumeric': 0.16282295821050807, 'fraction_numerical': 0.0041616091555401425, 'mean_word_length': 3.5027322404371586, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '610367', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1705, 'n_words': 349}