<commit_before>var PWD = process.env.PWD,
    DEBUG = process.env.DEBUG,
    fs = Npm.require('fs'),
    path = Npm.require('path'),
    glob = Npm.require('glob')

fileLoader = {
  loadFiles: loadFiles,
  getJsFiles: getJsFiles,
  getCoffeeFiles: getCoffeeFiles,
  filterFiles: filterFiles,
  loadFile: loadFile
}

/**
 * Loads a Meteor app's javascript and coffeescript files.
 * Matches Meteor core's load order.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method loadFiles
 * @param {Object} context Global context
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 */
function loadFiles (context, options) {
  var files = _.union(getJsFiles(options), getCoffeeFiles(options))

  files.sort(loadOrderSort([]))
  _.each(files, function (file) {
    loadFile(file, context)
  })
}

/**
 * Returns list of javascript filenames in Meteor app.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method getJsFiles
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array.<String>} list of filenames
 */
function getJsFiles (options) {
  var files = glob.sync('**/*.js', { cwd: PWD })

  return filterFiles(files, options)
}

/**
 * Returns list of coffeescript files in Meteor app.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method getCoffeeFiles
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array.<String>} list of filenames
 */
function getCoffeeFiles (options) {
  var files = glob.sync('**/*.coffee', { cwd: PWD })

  return filterFiles(files, options)
}

/**
 * Filters out any files in the following directories:
 *   private,
 *   public,
 *   programs,
 *   packages,
 *   tests
 *
 * @method filterFiles
 * @param {Array} files array of filenames to filter
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array} filenames
 */
function filterFiles (files, options) {
  var shouldIgnore = ['tests', 'private', 'public', 'programs', 'packages']

  options = options || {}

  if (options.ignoreDirs) {
    if ('string' === typeof options.ignoreDirs) {
      shouldIgnore.push(options.ignoreDirs)
    } else if (_.isArray(options.ignoreDirs)) {
      shouldIgnore = shouldIgnore.concat(options.ignoreDirs)
    }
  }

  return _.filter(files, function (filepath) {
    return !_.some(shouldIgnore, function (dirName) {
      var startPath = filepath.substring(0, dirName.length)
      return startPath === dirName
    })
  })
}

/**
 * Load and execute the target source file.
 * Will use node's 'require' if source file has a .js extension or
 * karma's coffeescript preprocessor if a .coffee extension
 *
 * @method loadFile
 * @param {String} target file path to load, relative to meteor app
 */
function loadFile (target, context) {
  var pwd = process.env.PWD,
      filename = path.join(pwd, target),
      ext

  if (fs.existsSync(filename)) {
    ext = path.extname(filename)
    if ('.js' === ext) {
      DEBUG && console.log('loading source file:', filename)
      runFileInContext(filename, context)
    } else if ('.coffee' === ext) {
      DEBUG && console.log('loading source file:', filename)
      coffeeRequire(filename)
    }
  }
}
<commit_msg>Quick fix for unit tests no longer running (fixes #28)<commit_after>var PWD = process.env.PWD,
    DEBUG = process.env.DEBUG,
    fs = Npm.require('fs'),
    path = Npm.require('path'),
    glob = Npm.require('glob')

fileLoader = {
  loadFiles: loadFiles,
  getJsFiles: getJsFiles,
  getCoffeeFiles: getCoffeeFiles,
  filterFiles: filterFiles,
  loadFile: loadFile
}

/**
 * Loads a Meteor app's javascript and coffeescript files.
 * Matches Meteor core's load order.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method loadFiles
 * @param {Object} context Global context
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 */
function loadFiles (context, options) {
  var files = _.union(getJsFiles(options), getCoffeeFiles(options))

  files.sort(loadOrderSort([]))
  _.each(files, function (file) {
    loadFile(file, context)
  })
}

/**
 * Returns list of javascript filenames in Meteor app.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method getJsFiles
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array.<String>} list of filenames
 */
function getJsFiles (options) {
  var files = glob.sync('**/*.js', { cwd: PWD })

  return filterFiles(files, options)
}

/**
 * Returns list of coffeescript files in Meteor app.
 *
 * Excluded directories: private, public, programs, packages, tests
 *
 * @method getCoffeeFiles
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array.<String>} list of filenames
 */
function getCoffeeFiles (options) {
  var files = glob.sync('**/*.coffee', { cwd: PWD })

  return filterFiles(files, options)
}

/**
 * Filters out any files in the following directories:
 *   private,
 *   public,
 *   programs,
 *   packages,
 *   tests
 *
 * @method filterFiles
 * @param {Array} files array of filenames to filter
 * @param {Object} [options]
 * @param {Array|String} [options.ignoreDirs] Directories to ignore
 * @return {Array} filenames
 */
function filterFiles (files, options) {
  var shouldIgnore = ['tests', 'private', 'public', 'programs', 'packages']

  options = options || {}

  if (options.ignoreDirs) {
    if ('string' === typeof options.ignoreDirs) {
      shouldIgnore.push(options.ignoreDirs)
    } else if (_.isArray(options.ignoreDirs)) {
      shouldIgnore = shouldIgnore.concat(options.ignoreDirs)
    }
  }

  return _.filter(files, function (filepath) {
    return !_.some(shouldIgnore, function (dirName) {
      var startPath = filepath.substring(0, dirName.length)
      return startPath === dirName
    })
  })
}

/**
 * Load and execute the target source file.
 * Will use node's 'require' if source file has a .js extension or
 * karma's coffeescript preprocessor if a .coffee extension
 *
 * @method loadFile
 * @param {String} target file path to load, relative to meteor app
 */
function loadFile (target, context) {
  var pwd = process.env.PWD,
      filename = path.resolve(pwd, target),
      ext

  if (fs.existsSync(filename)) {
    ext = path.extname(filename)
    if ('.js' === ext) {
      DEBUG && console.log('loading source file:', filename)
      runFileInContext(filename, context)
    } else if ('.coffee' === ext) {
      DEBUG && console.log('loading source file:', filename)
      coffeeRequire(filename)
    }
  }
}

{'fraction_non_alphanumeric': 0.13291324736225088, 'fraction_numerical': 0.0005861664712778429, 'mean_word_length': 3.792837078651685, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1429708', 'n_tokens_mistral': 2179, 'n_tokens_neox': 2122, 'n_words': 619}