<commit_before>var _ = require('lodash');
var fs = require('fs');
var path = require('path');
var RSVP = require('rsvp');
var quickTemp = require('quick-temp');
var mkdirp = require('mkdirp');
var browserify = require('browserify');
var walkSync  = require('walk-sync');
var hashTree = require('broccoli-kitchen-sink-helpers').hashTree;

function FastBrowserify(inputTree, options) {
  if (!(this instanceof FastBrowserify)) { return new FastBrowserify(inputTree, options); }

  this.options = _.extend(this.getDefaultOptions(), options);
  this.destDir = quickTemp.makeOrRemake(this, 'tmpDestDir');

  this.inputTree = inputTree;

  this.bundles = {};
  this.cache = {};
  this.watchFiles = {};
  this.packageCache = {};
}

FastBrowserify.prototype.getDefaultOptions = function() {
  return {
    browserify: {},
    bundleExtension: '.js.browserify',
    outputExtension: '.js'
  };
};

FastBrowserify.prototype.cleanup = function() {
  quickTemp.remove(this, 'tmpDestDir');
};

FastBrowserify.prototype.read = function(readTree) {
  var self = this;
  var promises = [];

  return readTree(this.inputTree).then(function(srcDir) {
    var paths = walkSync(srcDir);
    var extensionRegex = new RegExp(self.options.bundleExtension.replace(/\./, '\\.') + '$');
    var bundleFiles = paths.filter(function(relativePath) { return relativePath.match(extensionRegex); });

    // remove output files that don't have a corrisponding input file anymore
    self.cleanupBundles();
    var invalidatedBundles = self.invalidateCache();

    bundleFiles.forEach(function(relativePath) {
      var dir = path.dirname(relativePath);
      var bundle = self.bundles[relativePath];

      if (! bundle) {
        var outputBasename = relativePath.replace(extensionRegex, self.options.outputExtension);
        var outputAbsolutePath = path.resolve(srcDir, relativePath);

        bundle = {
          browserify: null,
          inputFileName: path.join(srcDir, relativePath),
          outputBasename: outputBasename,
          outputFileName: path.join(self.destDir, outputBasename),
          browserifyOptions: {},
          dependentFileNames: {}
        };

        bundle.browserifyOptions = _.extend(bundle.browserifyOptions, {
          basedir: srcDir,
          cache: self.cache,
          packageCache: self.packageCache,
          fullPaths: true,
          extensions: ['.js', self.options.bundleExtension],
          entries: [outputAbsolutePath]
        });

        bundle.browserify = browserify(bundle.browserifyOptions);

        bundle.browserify.on('dep', function(dep) {
          if (typeof dep.id == 'string') {
            bundle.browserifyOptions.cache[dep.id] = dep;
          }
          if (typeof dep.file == 'string') {
            var file = dep.file;
            if (file[0] == '.') {
              file = path.resolve(dep.basedir, file);
            }
            self.watchFiles[file] = hashTree(file);
            bundle.dependentFileNames[file] = file;
          }
        });

        bundle.browserify.on('file', function(file) {
          self.watchFiles[file] = hashTree(file);
          bundle.dependentFileNames[file] = file;
        });

        bundle.browserify.on('package', function(pkg) {
          var packageFile = path.join(pkg.__dirname, 'package.json');
          self.watchFiles[packageFile] = hashTree(packageFile);
          bundle.dependentFileNames[packageFile] = packageFile;
        });

        self.bundles[relativePath] = bundle;
      } else {
        // if this browserify bundle hasn't been invalidated
        // skip this file
        if (! _.include(invalidatedBundles, bundle)) {
          console.log('Skipping bundle', bundle.outputBasename);
          return;
        }
      }

      // Create the target directory in the destination
      mkdirp.sync(path.join(self.destDir, dir));

      promise = self.bundle(bundle);
      promises.push(promise);
    });

    return RSVP.all(promises).then(function(outputFiles) {
      return self.destDir;
    });
  });
};

FastBrowserify.prototype.invalidateCache = function() {
  var outputFileMTimes = {};
  var bundle;
  var bundleKey;<commit_msg>remove console.log<commit_after>var _ = require('lodash');
var fs = require('fs');
var path = require('path');
var RSVP = require('rsvp');
var quickTemp = require('quick-temp');
var mkdirp = require('mkdirp');
var browserify = require('browserify');
var walkSync  = require('walk-sync');
var hashTree = require('broccoli-kitchen-sink-helpers').hashTree;

function FastBrowserify(inputTree, options) {
  if (!(this instanceof FastBrowserify)) { return new FastBrowserify(inputTree, options); }

  this.options = _.extend(this.getDefaultOptions(), options);
  this.destDir = quickTemp.makeOrRemake(this, 'tmpDestDir');

  this.inputTree = inputTree;

  this.bundles = {};
  this.cache = {};
  this.watchFiles = {};
  this.packageCache = {};
}

FastBrowserify.prototype.getDefaultOptions = function() {
  return {
    browserify: {},
    bundleExtension: '.js.browserify',
    outputExtension: '.js'
  };
};

FastBrowserify.prototype.cleanup = function() {
  quickTemp.remove(this, 'tmpDestDir');
};

FastBrowserify.prototype.read = function(readTree) {
  var self = this;
  var promises = [];

  return readTree(this.inputTree).then(function(srcDir) {
    var paths = walkSync(srcDir);
    var extensionRegex = new RegExp(self.options.bundleExtension.replace(/\./, '\\.') + '$');
    var bundleFiles = paths.filter(function(relativePath) { return relativePath.match(extensionRegex); });

    // remove output files that don't have a corrisponding input file anymore
    self.cleanupBundles();
    var invalidatedBundles = self.invalidateCache();

    bundleFiles.forEach(function(relativePath) {
      var dir = path.dirname(relativePath);
      var bundle = self.bundles[relativePath];

      if (! bundle) {
        var outputBasename = relativePath.replace(extensionRegex, self.options.outputExtension);
        var outputAbsolutePath = path.resolve(srcDir, relativePath);

        bundle = {
          browserify: null,
          inputFileName: path.join(srcDir, relativePath),
          outputBasename: outputBasename,
          outputFileName: path.join(self.destDir, outputBasename),
          browserifyOptions: {},
          dependentFileNames: {}
        };

        bundle.browserifyOptions = _.extend(bundle.browserifyOptions, {
          basedir: srcDir,
          cache: self.cache,
          packageCache: self.packageCache,
          fullPaths: true,
          extensions: ['.js', self.options.bundleExtension],
          entries: [outputAbsolutePath]
        });

        bundle.browserify = browserify(bundle.browserifyOptions);

        bundle.browserify.on('dep', function(dep) {
          if (typeof dep.id == 'string') {
            bundle.browserifyOptions.cache[dep.id] = dep;
          }
          if (typeof dep.file == 'string') {
            var file = dep.file;
            if (file[0] == '.') {
              file = path.resolve(dep.basedir, file);
            }
            self.watchFiles[file] = hashTree(file);
            bundle.dependentFileNames[file] = file;
          }
        });

        bundle.browserify.on('file', function(file) {
          self.watchFiles[file] = hashTree(file);
          bundle.dependentFileNames[file] = file;
        });

        bundle.browserify.on('package', function(pkg) {
          var packageFile = path.join(pkg.__dirname, 'package.json');
          self.watchFiles[packageFile] = hashTree(packageFile);
          bundle.dependentFileNames[packageFile] = packageFile;
        });

        self.bundles[relativePath] = bundle;
      } else {
        // if this browserify bundle hasn't been invalidated
        // skip this file
        if (! _.include(invalidatedBundles, bundle)) {
          return;
        }
      }

      // Create the target directory in the destination
      mkdirp.sync(path.join(self.destDir, dir));

      promise = self.bundle(bundle);
      promises.push(promise);
    });

    return RSVP.all(promises).then(function(outputFiles) {
      return self.destDir;
    });
  });
};

FastBrowserify.prototype.invalidateCache = function() {
  var outputFileMTimes = {};
  var bundle;
  var bundleKey;
{'fraction_non_alphanumeric': 0.13130455868089233, 'fraction_numerical': 0.00024248302618816683, 'mean_word_length': 2.847481343283582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1383164', 'n_tokens_mistral': 2354, 'n_tokens_neox': 2317, 'n_words': 453}