<commit_before>'use strict';

var fs = require('fs');
var RSVP = require('rsvp');
var path = require('path');
var rimraf = RSVP.denodeify(require('rimraf'));
var mkdirp = require('mkdirp');
var walkSync = require('walk-sync');
var CoreObject = require('core-object');
var symlinkOrCopy = require('symlink-or-copy');
var generateRandomString = require('./lib/generate-random-string');

function makeDictionary() {
  var cache = Object.create(null);

  cache['_dict'] = null;
  delete cache['_dict'];

  return cache;
}

function Funnel(inputTree, options) {
  this.inputTree = inputTree;
  this._includeFileCache = makeDictionary();
  this._tmpDir = path.resolve(path.join(this.tmpRoot, 'funnel-dest_' + generateRandomString(6) + '.tmp'));

  for (var key in options) {
    if (options.hasOwnProperty(key)) {
      this[key] = options[key];
    }
  }

  this.setupDestPaths();

  if (this.include && !Array.isArray(this.include)) {
    throw new Error('Invalid include option, it must be an array.');
  }

  if (this.exclude && !Array.isArray(this.exclude)) {
    throw new Error('Invalid exclude option, it must be an array.');
  }
}

Funnel.__proto__ = CoreObject;
Funnel.prototype.constructor = Funnel;

Funnel.prototype.tmpRoot = 'tmp';

Funnel.prototype.setupDestPaths = function() {
  this.destDir = this.destDir || '/';
  this.destPath = path.join(this._tmpDir, this.destDir);

  if (this.destPath[this.destPath.length -1] === '/') {
    this.destPath = this.destPath.slice(0, -1);
  }
};

Funnel.prototype.shouldLinkRoots = function() {
  return !this.include && !this.exclude;
};<commit_msg>Use Object.keys instead of for in loop.<commit_after>'use strict';

var fs = require('fs');
var RSVP = require('rsvp');
var path = require('path');
var rimraf = RSVP.denodeify(require('rimraf'));
var mkdirp = require('mkdirp');
var walkSync = require('walk-sync');
var CoreObject = require('core-object');
var symlinkOrCopy = require('symlink-or-copy');
var generateRandomString = require('./lib/generate-random-string');

function makeDictionary() {
  var cache = Object.create(null);

  cache['_dict'] = null;
  delete cache['_dict'];

  return cache;
}

function Funnel(inputTree, options) {
  this.inputTree = inputTree;
  this._includeFileCache = makeDictionary();
  this._tmpDir = path.resolve(path.join(this.tmpRoot, 'funnel-dest_' + generateRandomString(6) + '.tmp'));

  var keys = Object.keys(options || {});
  for (var i = 0, l = keys.length; i < l; i++) {
    var key = keys[i];
    this[key] = options[key];
  }

  this.setupDestPaths();

  if (this.include && !Array.isArray(this.include)) {
    throw new Error('Invalid include option, it must be an array.');
  }

  if (this.exclude && !Array.isArray(this.exclude)) {
    throw new Error('Invalid exclude option, it must be an array.');
  }
}

Funnel.__proto__ = CoreObject;
Funnel.prototype.constructor = Funnel;

Funnel.prototype.tmpRoot = 'tmp';

Funnel.prototype.setupDestPaths = function() {
  this.destDir = this.destDir || '/';
  this.destPath = path.join(this._tmpDir, this.destDir);

  if (this.destPath[this.destPath.length -1] === '/') {
    this.destPath = this.destPath.slice(0, -1);
  }
};

Funnel.prototype.shouldLinkRoots = function() {
  return !this.include && !this.exclude;
};
{'fraction_non_alphanumeric': 0.16773995706838393, 'fraction_numerical': 0.0027598896044158236, 'mean_word_length': 4.35632183908046, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5156224', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1072, 'n_words': 238}