<commit_before>var plumber = require('gulp-plumber');
var gulpMocha = require('gulp-spawn-mocha');
var filter = require('gulp-filter');
var gulpFile = require('gulp-file');
var gulpIf = require('gulp-if');
var gulpWatch = require('gulp-watch');
var yargs = require('yargs');
var argv = require('yargs').argv;
var moment = require('moment');
var fs = require('fs');

/**
 * test task arguments
 * grep: grep string
 * d (boolean): allow debugger breakpoints (using node-inspector)
 * i (boolean): generate istanbul code coverate reports
 * clean: clean out test dir contents
 */

module.exports = function(gulp, options) {
  options = options || {};
  var scriptPathPrefix = (options.scriptsDir || 'js') + '/';
  var testFileSuffix = options.testFileSuffix || '-test';
  var testsDirName = options.testsDirName || '_tests';
  var testsInitFileName = options.testsInitFileName || '_init';

  gulp.task('test-js', function() {
    var mochaArgs = getMochaArgs();
    var clean = mochaArgs.c || mochaArgs.clean;
    mochaArgs.c = true;
    var watch = mochaArgs.w || mochaArgs.watch;
    mochaArgs.w = mochaArgs.watch = undefined;

    if (clean) {
      // clean the tests
      deleteFolderRecursive('.test');
    }

    var newCodeFilter = filter(function(file) {
      if (file.stat.isDirectory()) {
        return true;
      }
      if (hasBeenModified(file)) {
        return true;
      }
    });

    function loadTemplate(fileName) {
      return fs.readFileSync(__dirname + '/' + fileName, {encoding: 'utf8'})
        .replace('{scriptPathPrefix}', scriptPathPrefix)
        .replace('{testFileSuffix}', testFileSuffix)
        .replace('{testsDirName}', testsDirName)
        .replace('{testsInitFileName}', testsInitFileName);
    }

    // run mocha tests against all files
    function testAllFiles() {
      fs.writeFileSync(
        '.test/test-all.js',
        loadTemplate('test-runner.js'),
        {encoding: 'utf8'});

      mocha('.test/test-all.js', mochaArgs, watch);
    }

    // run mocha tests against a single file
    function testFile(file) {
      var path = file.path;
      // this should always be a test file
      var match = path.match(new RegExp('\/' + testsDirName + '\/[^\/]*'));
      if (!match) {
        path = path.replace(/(\/[^\/]*)$/, '/' + testsDirName + '$1', '').replace('.js', testFileSuffix + '.js');
      }

      fs.writeFileSync(
        './.test/test-single.js',
        loadTemplate('test-single-runner.js')
          .replace('{filePath}', path),
        {encoding: 'utf8'});

      mocha('./.test/test-single.js', mochaArgs, watch);
    }


    // create the gulp stream handlers
    var parts = [
      gulp.src('js/**/*')
    ];

    if (watch) {
      // we can't use the gulp stream to run tests if we're watching because
      // the end event is never received
      parts.push(gulpWatch('js/**/*', batch(function(files) {
        if (files.length > 1) {
          testAllFiles();
        } else {
          testFile(files[0]);
        }
      })));
      parts.push(plumber());
    }
    parts.push(newCodeFilter);
    parts.push(gulp.dest('.test/js'));

    if (!watch) {
      // get rid of all existing files
      parts.push(filter('_nada_'));
      // execute the mocha test as part of the standard gulp stream so it will
      // hook up correctly to other tasks
      parts.push(gulpFile('test-all.js', loadTemplate('test-runner.js')));
      parts.push(gulp.dest('.test'));
      parts.push(gulpMocha(mochaArgs));
    }

    var combiner = require('stream-combiner2');
    var combined = combiner.obj(parts);
    combined.on('error', function(err) {
      console.error(err.stack);
      process.exit(1);
    });
    return combined;

    function getMochaArgs() {
      var mochaArgs = {};
      var allowedArgs = ['A', 'c', 'C', 'G', 'R', 'S', 'b', 'd', 'g', 'i', 'r', 's', 't', 'u', 'w',
          'asyncOnly', 'colors', 'noColors', 'growl', 'reporter', 'sort', 'bail', 'debug', 'grep',
          'invert', 'require', 'slow', 'timeout', 'ui', 'watch', 'checkLeaks', 'compilers', 'debugBrk', 'globals',
          'harmony', 'harmonyCollections', 'harmonyGenerators', 'harmonyProxies', 'inlineDiffs', 'interfaces',<commit_msg>add additional pipe support<commit_after>var plumber = require('gulp-plumber');
var gulpMocha = require('gulp-spawn-mocha');
var filter = require('gulp-filter');
var gulpFile = require('gulp-file');
var gulpIf = require('gulp-if');
var gulpWatch = require('gulp-watch');
var yargs = require('yargs');
var argv = require('yargs').argv;
var moment = require('moment');
var fs = require('fs');

/**
 * test task arguments
 * grep: grep string
 * d (boolean): allow debugger breakpoints (using node-inspector)
 * i (boolean): generate istanbul code coverate reports
 * clean: clean out test dir contents
 */

module.exports = function(gulp, options) {
  options = options || {};
  var scriptPathPrefix = (options.scriptsDir || 'js') + '/';
  var testFileSuffix = options.testFileSuffix || '-test';
  var testsDirName = options.testsDirName || '_tests';
  var testsInitFileName = options.testsInitFileName || '_init';

  gulp.task('test-js', function() {
    var mochaArgs = getMochaArgs();
    var clean = mochaArgs.c || mochaArgs.clean;
    mochaArgs.c = true;
    var watch = mochaArgs.w || mochaArgs.watch;
    mochaArgs.w = mochaArgs.watch = undefined;

    if (clean) {
      // clean the tests
      deleteFolderRecursive('.test');
    }

    var newCodeFilter = filter(function(file) {
      if (file.stat.isDirectory()) {
        return true;
      }
      if (hasBeenModified(file)) {
        return true;
      }
    });

    function loadTemplate(fileName) {
      return fs.readFileSync(__dirname + '/' + fileName, {encoding: 'utf8'})
        .replace('{scriptPathPrefix}', scriptPathPrefix)
        .replace('{testFileSuffix}', testFileSuffix)
        .replace('{testsDirName}', testsDirName)
        .replace('{testsInitFileName}', testsInitFileName);
    }

    // run mocha tests against all files
    function testAllFiles() {
      fs.writeFileSync(
        '.test/test-all.js',
        loadTemplate('test-runner.js'),
        {encoding: 'utf8'});

      mocha('.test/test-all.js', mochaArgs, watch);
    }

    // run mocha tests against a single file
    function testFile(file) {
      var path = file.path;
      // this should always be a test file
      var match = path.match(new RegExp('\/' + testsDirName + '\/[^\/]*'));
      if (!match) {
        path = path.replace(/(\/[^\/]*)$/, '/' + testsDirName + '$1', '').replace('.js', testFileSuffix + '.js');
      }

      fs.writeFileSync(
        './.test/test-single.js',
        loadTemplate('test-single-runner.js')
          .replace('{filePath}', path),
        {encoding: 'utf8'});

      mocha('./.test/test-single.js', mochaArgs, watch);
    }


    // create the gulp stream handlers
    var parts = [
      gulp.src('js/**/*')
    ];

    if (watch) {
      // we can't use the gulp stream to run tests if we're watching because
      // the end event is never received
      parts.push(gulpWatch('js/**/*', batch(function(files) {
        if (files.length > 1) {
          testAllFiles();
        } else {
          testFile(files[0]);
        }
      })));
      parts.push(plumber());
    }
    parts.push(newCodeFilter);
    if (options.pipe && options.pipe.length) {
      Array.prototype.push.apply(parts, options.pipe);
    }

    parts.push(gulp.dest('.test/js'));

    if (!watch) {
      // get rid of all existing files
      parts.push(filter('_nada_'));
      // execute the mocha test as part of the standard gulp stream so it will
      // hook up correctly to other tasks
      parts.push(gulpFile('test-all.js', loadTemplate('test-runner.js')));
      parts.push(gulp.dest('.test'));
      parts.push(gulpMocha(mochaArgs));
    }

    var combiner = require('stream-combiner2');
    var combined = combiner.obj(parts);
    combined.on('error', function(err) {
      console.error(err.stack);
      process.exit(1);
    });
    return combined;

    function getMochaArgs() {
      var mochaArgs = {};
      var allowedArgs = ['A', 'c', 'C', 'G', 'R', 'S', 'b', 'd', 'g', 'i', 'r', 's', 't', 'u', 'w',
          'asyncOnly', 'colors', 'noColors', 'growl', 'reporter', 'sort', 'bail', 'debug', 'grep',
          'invert', 'require', 'slow', 'timeout', 'ui', 'watch', 'checkLeaks', 'compilers', 'debugBrk', 'globals',
          'harmony', 'harmonyCollections', 'harmonyGenerators', 'harmonyProxies', 'inlineDiffs', 'interfaces',
{'fraction_non_alphanumeric': 0.17161290322580644, 'fraction_numerical': 0.00187683284457478, 'mean_word_length': 2.8164726947179948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4449558', 'n_tokens_mistral': 2737, 'n_tokens_neox': 2695, 'n_words': 641}