<commit_before>'use strict';

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var Promise = require('rsvp').Promise;
var Plugin = require('broccoli-plugin');
var walkSync = require('walk-sync');
var mapSeries = require('promise-map-series');
var symlinkOrCopySync = require('symlink-or-copy').sync;
var debugGenerator = require('heimdalljs-logger');
var md5Hex = require('md5-hex');
var Processor = require('./lib/processor');
var defaultProccessor = require('./lib/strategies/default');
var hashForDep = require('hash-for-dep');
var FSTree = require('fs-tree-diff');
var heimdall = require('heimdalljs');
var queue = require('async-promise-queue');


function ApplyPatchesSchema() {
  this.mkdir = 0;
  this.rmdir = 0;
  this.unlink = 0;
  this.change = 0;
  this.create = 0;
  this.other = 0;
  this.processed = 0;
  this.linked = 0;

  this.processString = 0;
  this.processStringTime = 0;
  this.persistentCacheHit = 0;
  this.persistentCachePrime = 0;
}

function DerivePatchesSchema() {
  this.patches = 0;
  this.entries = 0;
}

module.exports = Filter;

Filter.prototype = Object.create(Plugin.prototype);
Filter.prototype.constructor = Filter;

function Filter(inputTree, options) {
  if (!this || !(this instanceof Filter) ||
      Object.getPrototypeOf(this) === Filter.prototype) {
    throw new TypeError('Filter is an abstract class and must be sub-classed');
  }

  var loggerName = 'broccoli-persistent-filter:' + (this.constructor.name);
  var annotation = (options && options.annotation) || this.annotation || this.description;

  if (annotation) {
    loggerName += ' > [' + annotation + ']';
  }

  this._logger = debugGenerator(loggerName);

  Plugin.call(this, [inputTree], {
    name: (options && options.name) || this.name || loggerName,
    annotation: (options && options.annotation) || this.annotation || annotation,
    persistentOutput: true
  });

  this.processor = new Processor(options);
  this.processor.setStrategy(defaultProccessor);
  this.currentTree = new FSTree();

  /* Destructuring assignment in node 0.12.2 would be really handy for this! */
  if (options) {
    if (options.extensions != null)      this.extensions = options.extensions;
    if (options.targetExtension != null) this.targetExtension = options.targetExtension;
    if (options.inputEncoding != null)   this.inputEncoding = options.inputEncoding;
    if (options.outputEncoding != null)  this.outputEncoding = options.outputEncoding;
    if (options.persist) {
      this.processor.setStrategy(require('./lib/strategies/persistent'));
    }
    this.async = (options.async === true);
  }

  this.processor.init(this);

  this._canProcessCache = Object.create(null);
  this._destFilePathCache = Object.create(null);
  this._needsReset = false;
}

function nanosecondsSince(time) {
  var delta = process.hrtime(time);
  return delta[0] * 1e9 + delta[1];
}

function timeSince(time) {
  var deltaNS = nanosecondsSince(time);
  return (deltaNS / 1e6).toFixed(2) +' ms';
}

Filter.prototype.build = function() {
  if (this._needsReset) {
    this.currentTree = new FSTree();
    rimraf.sync(this.outputPath);
    mkdirp.sync(this.outputPath);
  }

  this._needsReset = true;

  var srcDir = this.inputPaths[0];
  var destDir = this.outputPath;

  var prevTime = process.hrtime();
  var instrumentation = heimdall.start('derivePatches', DerivePatchesSchema);

  var walkStart = process.hrtime();
  var entries = walkSync.entries(srcDir);
  var walkDuration = timeSince(walkStart);

  var nextTree = FSTree.fromEntries(entries);
  var currentTree = this.currentTree;

  this.currentTree = nextTree;

  var patches = currentTree.calculatePatch(nextTree);

  instrumentation.stats.patches = patches.length;
  instrumentation.stats.entries = entries.length;
  instrumentation.stats.walk = {
    entries: entries.length,
    duration: walkDuration
  };

  this._logger.info('derivePatches', 'duration:', timeSince(prevTime), JSON.stringify(instrumentation.stats));

  instrumentation.stop();
  var plugin = this;

  // used with options.async = true to allow 'create' and 'change' operations to complete async
  var pendingWork = [];

  return new Promise(function(resolve) {
    resolve(heimdall.node('applyPatches', ApplyPatchesSchema, function(instrumentation) {
      var prevTime = process.hrtime();
      return new Promise(function(resolve) {
        var result = mapSeries(patches, function(patch) {
          var operation = patch[0];
          var relativePath = patch[1];
          var entry = patch[2];
          var outputPath = destDir + '/' + (plugin.getDestFilePath(relativePath, entry) || relativePath);
          var outputFilePath = outputPath;

          plugin._logger.debug('[operation:%s] %s', operation, relativePath);

          switch (operation) {
            case 'mkdir': {
              instrumentation.mkdir++;
              return fs.mkdirSync(outputPath);
            } case 'rmdir': {
              instrumentation.rmdir++;
              return fs.rmdirSync(outputPath);
            } case 'unlink': {
              instrumentation.unlink++;
              return fs.unlinkSync(outputPath);
            } case 'change': {
              instrumentation.change++;
              var changeOperation = plugin._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, true, instrumentation);
              // will be undefined if this.canProcessFile() == false
              if (plugin.async && changeOperation !== undefined) {
                pendingWork.push(changeOperation);
                return;
              }
              return changeOperation;
            } case 'create': {
              instrumentation.create++;
              var createOperation = plugin._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, false, instrumentation);
              // will be undefined if this.canProcessFile() == false
              if (plugin.async && createOperation !== undefined) {
                pendingWork.push(createOperation);
                return;
              }
              return createOperation;
            } default: {
              instrumentation.other++;
            }
          }
        });
        resolve(result);
      }).then(() => {
        const worker = queue.async.asyncify((promise) => promise);
        return queue(worker, pendingWork, 4);
      }).then((result) => {
        plugin._logger.info('applyPatches', 'duration:', timeSince(prevTime), JSON.stringify(instrumentation));
        plugin._needsReset = false;
        return result;
      });
    }));
  });
};

Filter.prototype._handleFile = function(relativePath, srcDir, destDir, entry, outputPath, isChange, instrumentation) {
  if (this.canProcessFile(relativePath, entry)) {
    instrumentation.processed++;
    return this.processAndCacheFile(srcDir, destDir, entry, isChange, instrumentation);
  } else {
    instrumentation.linked++;
    if (isChange) {
      fs.unlinkSync(outputPath);
    }
    var srcPath = srcDir + '/' + relativePath;
    return symlinkOrCopySync(srcPath, outputPath);
  }
};

/*
 The cache key to be used for this plugins set of dependencies. By default
 a hash is created based on `package.json` and nested dependencies.

 Implement this to customize the cache key (for example if you need to<commit_msg>configurable concurrency, and simplify applyPatches section<commit_after>'use strict';

var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var Promise = require('rsvp').Promise;
var Plugin = require('broccoli-plugin');
var walkSync = require('walk-sync');
var mapSeries = require('promise-map-series');
var symlinkOrCopySync = require('symlink-or-copy').sync;
var debugGenerator = require('heimdalljs-logger');
var md5Hex = require('md5-hex');
var Processor = require('./lib/processor');
var defaultProccessor = require('./lib/strategies/default');
var hashForDep = require('hash-for-dep');
var FSTree = require('fs-tree-diff');
var heimdall = require('heimdalljs');
var queue = require('async-promise-queue');

var concurrency = Number(process.env.JOBS) || require('os').cpus().length;

function ApplyPatchesSchema() {
  this.mkdir = 0;
  this.rmdir = 0;
  this.unlink = 0;
  this.change = 0;
  this.create = 0;
  this.other = 0;
  this.processed = 0;
  this.linked = 0;

  this.processString = 0;
  this.processStringTime = 0;
  this.persistentCacheHit = 0;
  this.persistentCachePrime = 0;
}

function DerivePatchesSchema() {
  this.patches = 0;
  this.entries = 0;
}

module.exports = Filter;

Filter.prototype = Object.create(Plugin.prototype);
Filter.prototype.constructor = Filter;

function Filter(inputTree, options) {
  if (!this || !(this instanceof Filter) ||
      Object.getPrototypeOf(this) === Filter.prototype) {
    throw new TypeError('Filter is an abstract class and must be sub-classed');
  }

  var loggerName = 'broccoli-persistent-filter:' + (this.constructor.name);
  var annotation = (options && options.annotation) || this.annotation || this.description;

  if (annotation) {
    loggerName += ' > [' + annotation + ']';
  }

  this._logger = debugGenerator(loggerName);

  Plugin.call(this, [inputTree], {
    name: (options && options.name) || this.name || loggerName,
    annotation: (options && options.annotation) || this.annotation || annotation,
    persistentOutput: true
  });

  this.processor = new Processor(options);
  this.processor.setStrategy(defaultProccessor);
  this.currentTree = new FSTree();

  /* Destructuring assignment in node 0.12.2 would be really handy for this! */
  if (options) {
    if (options.extensions != null)      this.extensions = options.extensions;
    if (options.targetExtension != null) this.targetExtension = options.targetExtension;
    if (options.inputEncoding != null)   this.inputEncoding = options.inputEncoding;
    if (options.outputEncoding != null)  this.outputEncoding = options.outputEncoding;
    if (options.persist) {
      this.processor.setStrategy(require('./lib/strategies/persistent'));
    }
    this.async = (options.async === true);
  }

  this.processor.init(this);

  this._canProcessCache = Object.create(null);
  this._destFilePathCache = Object.create(null);
  this._needsReset = false;
}

function nanosecondsSince(time) {
  var delta = process.hrtime(time);
  return delta[0] * 1e9 + delta[1];
}

function timeSince(time) {
  var deltaNS = nanosecondsSince(time);
  return (deltaNS / 1e6).toFixed(2) +' ms';
}

Filter.prototype.build = function() {
  if (this._needsReset) {
    this.currentTree = new FSTree();
    rimraf.sync(this.outputPath);
    mkdirp.sync(this.outputPath);
  }

  this._needsReset = true;

  var srcDir = this.inputPaths[0];
  var destDir = this.outputPath;

  var prevTime = process.hrtime();
  var instrumentation = heimdall.start('derivePatches', DerivePatchesSchema);

  var walkStart = process.hrtime();
  var entries = walkSync.entries(srcDir);
  var walkDuration = timeSince(walkStart);

  var nextTree = FSTree.fromEntries(entries);
  var currentTree = this.currentTree;

  this.currentTree = nextTree;

  var patches = currentTree.calculatePatch(nextTree);

  instrumentation.stats.patches = patches.length;
  instrumentation.stats.entries = entries.length;
  instrumentation.stats.walk = {
    entries: entries.length,
    duration: walkDuration
  };

  this._logger.info('derivePatches', 'duration:', timeSince(prevTime), JSON.stringify(instrumentation.stats));

  instrumentation.stop();
  var plugin = this;

  // used with options.async = true to allow 'create' and 'change' operations to complete async
  var pendingWork = [];

  return Promise.resolve(heimdall.node('applyPatches', ApplyPatchesSchema, function(instrumentation) {
    var prevTime = process.hrtime();
    return new Promise(function(resolve) {
      var result = mapSeries(patches, function(patch) {
        var operation = patch[0];
        var relativePath = patch[1];
        var entry = patch[2];
        var outputPath = destDir + '/' + (plugin.getDestFilePath(relativePath, entry) || relativePath);
        var outputFilePath = outputPath;

        plugin._logger.debug('[operation:%s] %s', operation, relativePath);

        switch (operation) {
          case 'mkdir': {
            instrumentation.mkdir++;
            return fs.mkdirSync(outputPath);
          } case 'rmdir': {
            instrumentation.rmdir++;
            return fs.rmdirSync(outputPath);
          } case 'unlink': {
            instrumentation.unlink++;
            return fs.unlinkSync(outputPath);
          } case 'change': {
            instrumentation.change++;
            var changeOperation = plugin._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, true, instrumentation);
            // will be undefined if this.canProcessFile() == false
            if (plugin.async && changeOperation !== undefined) {
              pendingWork.push(changeOperation);
              return;
            }
            return changeOperation;
          } case 'create': {
            instrumentation.create++;
            var createOperation = plugin._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, false, instrumentation);
            // will be undefined if this.canProcessFile() == false
            if (plugin.async && createOperation !== undefined) {
              pendingWork.push(createOperation);
              return;
            }
            return createOperation;
          } default: {
            instrumentation.other++;
          }
        }
      });
      resolve(result);
    }).then(() => {
      const worker = queue.async.asyncify((promise) => promise);
      return queue(worker, pendingWork, concurrency);
    }).then((result) => {
      plugin._logger.info('applyPatches', 'duration:', timeSince(prevTime), JSON.stringify(instrumentation));
      plugin._needsReset = false;
      return result;
    });
  }));
};

Filter.prototype._handleFile = function(relativePath, srcDir, destDir, entry, outputPath, isChange, instrumentation) {
  if (this.canProcessFile(relativePath, entry)) {
    instrumentation.processed++;
    return this.processAndCacheFile(srcDir, destDir, entry, isChange, instrumentation);
  } else {
    instrumentation.linked++;
    if (isChange) {
      fs.unlinkSync(outputPath);
    }
    var srcPath = srcDir + '/' + relativePath;
    return symlinkOrCopySync(srcPath, outputPath);
  }
};

/*
 The cache key to be used for this plugins set of dependencies. By default
 a hash is created based on `package.json` and nested dependencies.

 Implement this to customize the cache key (for example if you need to
{'fraction_non_alphanumeric': 0.1279885566378312, 'fraction_numerical': 0.004291260813296097, 'mean_word_length': 3.3220488666470414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604502', 'n_tokens_mistral': 4274, 'n_tokens_neox': 4173, 'n_words': 974}