<commit_before>var fs = require('fs'),
    Q = require('q'),
    path = require('path');

module.exports = function(dirsConfig, basedir) {
  // TODO: Make this asynchronous
  // TODO: Make this recursive
  var jsFilesNested = [];

  var dirQueue = [];

  var scanFolder = function(dirConfig, dirPath) {
    var files = fs.readdirSync(dirPath);

    var dirFiles = [];

    files.forEach(function(file) {
      var fullFilePath = path.join(dirPath, file);
      var stats = fs.statSync(fullFilePath);
      if (stats.isDirectory()) {
        Array.prototype.push.apply(dirFiles, scanFolder(dirConfig, fullFilePath));
      } else if (path.extname(file) === '.js') {
        var filepath = path.relative(basedir, fullFilePath);
        var relativeFilepath = path.relative(dirConfig.dir, filepath);
        filepath = filepath.split(path.sep).join('/');
        if (dirConfig.exclude.indexOf(relativeFilepath) === -1 &&
          dirConfig.first.indexOf(relativeFilepath) === -1 &&
          dirConfig.last.indexOf(relativeFilepath) === -1) {
          dirFiles.push(filepath);
        }
      }
    });

    return dirFiles;
  };

  dirsConfig.forEach(function(dirConfig) {
    var firstFiles = dirConfig.first.map(function(filename) {
      return dirConfig.dir + '/' + filename;
    });
    var lastFiles = dirConfig.last.map(function(filename) {
      return dirConfig.dir + '/' + filename;
    });

    jsFilesNested.push(firstFiles);
    jsFilesNested.push(scanFolder(dirConfig, path.join(basedir, dirConfig.dir)));
    jsFilesNested.push(lastFiles);
  });

  var jsFiles = [];
  jsFilesNested.forEach(function(files) {
    Array.prototype.push.apply(jsFiles, files);
  });

  return jsFiles;
};<commit_msg>We don't need no stinkin' closures!<commit_after>var fs = require('fs'),
    Q = require('q'),
    path = require('path');

module.exports = function(dirsConfig, basedir) {
  // TODO: Make this asynchronous
  var jsFilesNested = [];

  dirsConfig.forEach(function(dirConfig) {
    var firstFiles = dirConfig.first.map(function(filename) {
      return dirConfig.dir + '/' + filename;
    });
    var lastFiles = dirConfig.last.map(function(filename) {
      return dirConfig.dir + '/' + filename;
    });

    jsFilesNested.push(firstFiles);
    jsFilesNested.push(scanFolder(dirConfig, basedir, path.join(basedir, dirConfig.dir)));
    jsFilesNested.push(lastFiles);
  });

  var jsFiles = [];
  jsFilesNested.forEach(function(files) {
    Array.prototype.push.apply(jsFiles, files);
  });

  return jsFiles;
};

function scanFolder(dirConfig, basedir, dirPath) {
  var files = fs.readdirSync(dirPath);

  var dirFiles = [];

  files.forEach(function(file) {
    var fullFilePath = path.join(dirPath, file);
    var stats = fs.statSync(fullFilePath);
    if (stats.isDirectory()) {
      Array.prototype.push.apply(dirFiles, scanFolder(dirConfig, basedir, fullFilePath));
    } else if (path.extname(file) === '.js') {
      var filepath = path.relative(basedir, fullFilePath);
      var relativeFilepath = path.relative(dirConfig.dir, filepath);
      filepath = filepath.split(path.sep).join('/');
      if (dirConfig.exclude.indexOf(relativeFilepath) === -1 &&
        dirConfig.first.indexOf(relativeFilepath) === -1 &&
        dirConfig.last.indexOf(relativeFilepath) === -1) {
        dirFiles.push(filepath);
      }
    }
  });

  return dirFiles;
}
{'fraction_non_alphanumeric': 0.14870881567230632, 'fraction_numerical': 0.0017809439002671415, 'mean_word_length': 3.37094682230869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2186506', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1043, 'n_words': 187}