<commit_before>'use strict';

var async = require('async');
var isString = require('101/is-string');
var FsDriver = require('./fs-driver');
var Warning = require('./warning');
var ScriptGenerator = require('./script-generator');
var exists = require('101/exists');
var debug = require('debug');
var fullDiffDebug = debug('fs-transform:full-diff');

/**
 * @module fs-transform:Transformer
 * @author <NAME>
 */
module.exports = Transformer;

/**
 * Class that performs a series of filesystem tranformations given a root
 * directory and list of rules.
 * @class
 * @author <NAME>
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @throws SyntaxError If rules were provided as a string, but were not valid
 *   JSON.
 * @throws Error If the given rules were not an array.
 */
function Transformer(root, rules) {
  if (isString(rules)) {
    rules = JSON.parse(rules);
  }
  if (!Array.isArray(rules)) {
    throw new Error('Rules must be an array.');
  }

  this.driver = new FsDriver(root);
  this.rules = rules;
  this._ruleActions = {};
  this.currentResult = null;
  this.warnings = [];
  this.results = [];
  this.nameChanges = [];
  this._globalExcludes = [];
  this.scriptGenerator = new ScriptGenerator(rules);

  this.setAction('copy', this.copy);
  this.setAction('rename', this.rename);
  this.setAction('replace', this.replace);
  this.setAction('exclude', this.exclude);
}

/**
 * Default postfix to append to backup files after performing find and replace
 * transformation operations.
 * @type {string}
 */
Transformer.ORIGINAL_POSTFIX = '.fs-transform.original';

/**
 * Pushes a new result to the result list.
 * @param {object} rule Rule that generates the result.
 * @return {object} The new result object.
 */
Transformer.prototype.pushResult = function (rule) {
  this.currentResult = {
    rule: rule,
    warnings: [],
    nameChanges: [],
    diffs: {}
  };
  this.results.push(this.currentResult);
  return this.currentResult;
};

/**
 * Adds a rule generated warning to the transformer.
 * @param object Object that generated the warning.
 * @param msg Message for the warning.
 */
Transformer.prototype.addWarning = function (object, msg) {
  var warning = new Warning(object, msg);
  this.warnings.push(warning);
  if (exists(this.currentResult)) {
    this.currentResult.warnings.push(warning);
  }
};

/**
 * Adds a name change result.
 * @param {object} nameChange Name change to add.
 */
Transformer.prototype.addNameChange = function (from, to) {
  var nameChange = {
    from: from,
    to: to
  };
  this.nameChanges.push(nameChange);
  if (exists(this.currentResult)) {
    this.currentResult.nameChanges.push(nameChange);
  }
};

/**
 * Adds a diff to the current result.
 * @param {string} filename Name of the file for the diff.
 * @param {string} Contents of the diff (via diff -u, similar to git diff).
 */
Transformer.prototype.addDiff = function (filename, diff) {
  if (!this.currentResult) { return; }
  if (!this.currentResult.diffs[filename]) {
    this.currentResult.diffs[filename] = [];
  }
  this.currentResult.diffs[filename].push(diff);
};

/**
 * Generates a shell script from this transfromer's rules.
 * @return {string} A shell-script that executes the rules.
 */
Transformer.prototype.getScript = function () {
  return this.scriptGenerator.generate();
};

/**
 * @return The full diff after performing all filesystem transforms.
 */
Transformer.prototype.getDiff = function() {
  return this._fullDiff;
};

/**
 * Allows the user to override existing rule actions and define new ones.
 * Note the action function will be applied within the context of this
 * Transformation.
 * @param {string} name Name of the action to define or override.
 * @param {fs-transform~Rule} fn Function to apply when encountering the rule
 *   with the given action name.
 */
Transformer.prototype.setAction = function(name, fn) {
  var self = this;
  this._ruleActions[name] = function() {
    fn.apply(self, arguments);
  };
};

/**
 * Determine the rule action handler for the action name.
 * @param {string} name Name of the action.
 * @return {fs-transform~Rule} The rule handler for the action.
 */
Transformer.prototype.getAction = function(name) {
  return this._ruleActions[name];
};

/**
 * Callback for the application of rule actions.
 * @callback fs-transform~Rule
 * @param {object} rule Rule to apply.
 * @param {function} cb Callback to execute after the rule has been applied.
 */

/**
 * Performs a series of filesystem transformations.
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.transform = function(root, rules, cb) {
  try {
    new Transformer(root, rules).transform(cb);
  }
  catch (err) {
    cb(err);
  }
};

/**
 * Performs a dry run of the transformation.
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.dry = function(root, rules, cb) {
  try {
    new Transformer(root, rules).dry(cb);
  }
  catch (err) {
    cb(err);
  }
};

/**
 * Callback executed when transformations have been completed, or if an error
 * has occurred.
 * @callback fs-transform~Callback
 * @param {Error} err An error if something went wrong when processing the
 *   transformation rules.
 */

/**
 * Perform and commit the transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype.transform = function (cb) {
  this._execute(true, cb);
};

/**
 * Perform a dry run of of the tranformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype.dry = function (cb) {
  this._execute(false, cb);
};

/**
 * Executes each of the the tranformation rules and collects their results.
 * @param {boolean} commit Whether or not to commit the transformations to the
 *   original root directory. If true then the changes are applied to the root
 *   directory, if false then the root directory will remain unchanged.
 * @param {fs-transform~Callback} executeCallback Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype._execute = function (commit, executeCallback) {
  var self = this;
  this._fullDiff = "";

  async.series([
    // 0. Check to ensure we have all the required commands
    function checkRequiredCommands(cb) {
      self.driver.hasAllCommands(cb);
    },

    // 1. Create a working directory
    function setup(cb) {
      self.driver.createWorkingDirectory(cb);
    },

    // 2. Apply transformation rules
    function applyRules(cb) {
      async.mapSeries(self.rules, function (rule, ruleCallback) {
        self.applyRule(rule, ruleCallback);
      }, function (err) {
        cb(err, self);
      });
    },

    // 3. Fetch a diff between the working and the original root
    function fetchFullDiff(cb) {
      self.driver.workingDiff(function (err, diff) {
        if (err) { return cb(err); }
        fullDiffDebug(diff);
        self._fullDiff = diff;
        cb();
      });
    },

    // 4. Commit the changes if applicable, otherwise remove the working
    //    directory.
    function cleanup(cleanupCallback) {
      if (!commit) {
        return self.driver.removeWorkingDirectory(cleanupCallback);
      }

      var root = self.driver.root;
      var backup = root + '.bak';
      var working = self.driver.working;

      async.series([
        function backupRoot(cb) {
          self.driver.move(root, backup, cb);
        },
        function moveWorkingToRoot(cb) {
          self.driver.move(working, root, cb);
        },
        function removeBackup(cb) {
          self.driver.removeRecursive(backup, cb);
        }
      ], cleanupCallback);
    }
  ], function (err) {
    executeCallback(err, self);
  });
};

/**
 * Applys a given transformation rule.
 * @param {object} rule Rule to apply.
 * @param {function} cb Callback to execute once the rule has been processed.
 */
Transformer.prototype.applyRule = function (rule, cb) {
  if (!rule) {
    return cb(new Error('Null or undfined rule encountered.'));
  }

  if (!isString(rule.action)) {
    return cb(new Error('Invalid rule action: ' + rule.action));
  }

  var actionMethod = this.getAction(rule.action);
  if (!actionMethod) {
    return cb(new Error('Action not found: ' + rule.action));
  }

  this.pushResult(rule);
  actionMethod(rule, cb);
};

/**
 * Performs validations for copy and rename rules.
 * @param {object} rule Rule to validate.
 * @return {boolean} `true` if validations pass, `false` otherwise.
 */
Transformer.prototype._isValidRenameOrCopy = function (rule) {
  if (!isString(rule.source)) {
    this.addWarning(rule, 'Missing source file.');
    return false;
  }
  if (!isString(rule.dest)) {
    this.addWarning(rule, 'Missing destination file.');
    return false;
  }
  if (!this.driver.exists(rule.source)) {
    this.addWarning(rule, 'Source file does not exist.');
    return false;
  }
  if (this.driver.exists(rule.dest)) {
    this.addWarning(rule, 'Overwrites destination file.');
  }
  return true;
};

/**
 * Copy a file.
 * @param {object} rule Defines the source and destination for the file copy.
 * @param {function} cb Callback to execute once the file has been copied.
 */
Transformer.prototype.copy = function (rule, cb) {
  if (!this._isValidRenameOrCopy(rule)) {
    return cb();
  }
  var self = this;
  this.driver.copy(rule.source, rule.dest, function (err, output) {
    if (err) { return cb(err); }
    self.addNameChange(rule.source, rule.dest);
    cb();
  });
};

/**
 * Rename a file.
 * @param {object} rule Defines the source and destination for the file rename.
 * @param {function} cb Callback to execute once the file has been renamed.
 */
Transformer.prototype.rename = function (rule, cb) {
  if (!this._isValidRenameOrCopy(rule)) {
    return cb();
  }
  var self = this;
  this.driver.move(rule.source, rule.dest, function (err, output) {
    if (err) { return cb(err); }
    self.addNameChange(rule.source, rule.dest);
    cb();
  });
};

/**
 * Text search and replace for many files.
 * @param {object} rule Rules that define what to search for, replace, and
 *   exclude from the search.
 * @param {function} cb Callback to execute once the search and replace has been
 *   performed.
 */
Transformer.prototype.replace = function (rule, cb) {
  var self = this;

  var search = rule.search;
  if (!isString(search)) {
    this.addWarning(rule, 'Search pattern not specified.');
    return cb();
  }

  var replace = rule.replace;
  if (!isString(replace)) {
    this.addWarning(rule, 'Replacement not specified.');
    return cb();
  }

  var exclude = rule.exclude || [];
  if (!Array.isArray(exclude)) {
    this.addWarning(rule, 'Excludes not supplied as an array, omitting.');
    exclude = [];
  }

  this.driver.grep(rule.search, function (err, grepResult) {
    var files = [];

    // 1. Determine files & lines from grep and filter out global excludes
    grepResult.split('\n').forEach(function (line) {
      var match = line.match(/(.*):(\d+):.*/);
      if (match) {
        files.push({ name: match[1], line: match[2] });
      }
    });

    // 1.1 If no files were returned by grep, issue a warning and return
    if (files.length === 0) {
      self.addWarning(rule, 'Search did not return any results.');
      return cb();
    }

    // 2. Remove excluded files/lines from result set

    // 2.0 Filter out global excludes
    files = files.filter(function (file) {
      return !~self._globalExcludes.indexOf(file.name);
    });

    // 2.1 Keep track of when excludes are used to filter results
    var ruleApplied = [];
    exclude.forEach(function(excludeRule, index) {
      ruleApplied[index] = false;
    });

    // 2.2 Handle local rule file excludes
    files = files.filter(function (file) {
      var keepFile = true;
      exclude.forEach(function(name, index) {<commit_msg>Fixed comments in replace routine.<commit_after>'use strict';

var async = require('async');
var isString = require('101/is-string');
var FsDriver = require('./fs-driver');
var Warning = require('./warning');
var ScriptGenerator = require('./script-generator');
var exists = require('101/exists');
var debug = require('debug');
var fullDiffDebug = debug('fs-transform:full-diff');

/**
 * @module fs-transform:Transformer
 * @author <NAME>
 */
module.exports = Transformer;

/**
 * Class that performs a series of filesystem tranformations given a root
 * directory and list of rules.
 * @class
 * @author <NAME>
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @throws SyntaxError If rules were provided as a string, but were not valid
 *   JSON.
 * @throws Error If the given rules were not an array.
 */
function Transformer(root, rules) {
  if (isString(rules)) {
    rules = JSON.parse(rules);
  }
  if (!Array.isArray(rules)) {
    throw new Error('Rules must be an array.');
  }

  this.driver = new FsDriver(root);
  this.rules = rules;
  this._ruleActions = {};
  this.currentResult = null;
  this.warnings = [];
  this.results = [];
  this.nameChanges = [];
  this._globalExcludes = [];
  this.scriptGenerator = new ScriptGenerator(rules);

  this.setAction('copy', this.copy);
  this.setAction('rename', this.rename);
  this.setAction('replace', this.replace);
  this.setAction('exclude', this.exclude);
}

/**
 * Default postfix to append to backup files after performing find and replace
 * transformation operations.
 * @type {string}
 */
Transformer.ORIGINAL_POSTFIX = '.fs-transform.original';

/**
 * Pushes a new result to the result list.
 * @param {object} rule Rule that generates the result.
 * @return {object} The new result object.
 */
Transformer.prototype.pushResult = function (rule) {
  this.currentResult = {
    rule: rule,
    warnings: [],
    nameChanges: [],
    diffs: {}
  };
  this.results.push(this.currentResult);
  return this.currentResult;
};

/**
 * Adds a rule generated warning to the transformer.
 * @param object Object that generated the warning.
 * @param msg Message for the warning.
 */
Transformer.prototype.addWarning = function (object, msg) {
  var warning = new Warning(object, msg);
  this.warnings.push(warning);
  if (exists(this.currentResult)) {
    this.currentResult.warnings.push(warning);
  }
};

/**
 * Adds a name change result.
 * @param {object} nameChange Name change to add.
 */
Transformer.prototype.addNameChange = function (from, to) {
  var nameChange = {
    from: from,
    to: to
  };
  this.nameChanges.push(nameChange);
  if (exists(this.currentResult)) {
    this.currentResult.nameChanges.push(nameChange);
  }
};

/**
 * Adds a diff to the current result.
 * @param {string} filename Name of the file for the diff.
 * @param {string} Contents of the diff (via diff -u, similar to git diff).
 */
Transformer.prototype.addDiff = function (filename, diff) {
  if (!this.currentResult) { return; }
  if (!this.currentResult.diffs[filename]) {
    this.currentResult.diffs[filename] = [];
  }
  this.currentResult.diffs[filename].push(diff);
};

/**
 * Generates a shell script from this transfromer's rules.
 * @return {string} A shell-script that executes the rules.
 */
Transformer.prototype.getScript = function () {
  return this.scriptGenerator.generate();
};

/**
 * @return The full diff after performing all filesystem transforms.
 */
Transformer.prototype.getDiff = function() {
  return this._fullDiff;
};

/**
 * Allows the user to override existing rule actions and define new ones.
 * Note the action function will be applied within the context of this
 * Transformation.
 * @param {string} name Name of the action to define or override.
 * @param {fs-transform~Rule} fn Function to apply when encountering the rule
 *   with the given action name.
 */
Transformer.prototype.setAction = function(name, fn) {
  var self = this;
  this._ruleActions[name] = function() {
    fn.apply(self, arguments);
  };
};

/**
 * Determine the rule action handler for the action name.
 * @param {string} name Name of the action.
 * @return {fs-transform~Rule} The rule handler for the action.
 */
Transformer.prototype.getAction = function(name) {
  return this._ruleActions[name];
};

/**
 * Callback for the application of rule actions.
 * @callback fs-transform~Rule
 * @param {object} rule Rule to apply.
 * @param {function} cb Callback to execute after the rule has been applied.
 */

/**
 * Performs a series of filesystem transformations.
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.transform = function(root, rules, cb) {
  try {
    new Transformer(root, rules).transform(cb);
  }
  catch (err) {
    cb(err);
  }
};

/**
 * Performs a dry run of the transformation.
 * @param {String} root Root directory to run the transformations.
 * @param {String|Array} rules An array of transformations, or a JSON string
 *   that parses into a stream of transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.dry = function(root, rules, cb) {
  try {
    new Transformer(root, rules).dry(cb);
  }
  catch (err) {
    cb(err);
  }
};

/**
 * Callback executed when transformations have been completed, or if an error
 * has occurred.
 * @callback fs-transform~Callback
 * @param {Error} err An error if something went wrong when processing the
 *   transformation rules.
 */

/**
 * Perform and commit the transformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype.transform = function (cb) {
  this._execute(true, cb);
};

/**
 * Perform a dry run of of the tranformations.
 * @param {fs-transform~Callback} cb Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype.dry = function (cb) {
  this._execute(false, cb);
};

/**
 * Executes each of the the tranformation rules and collects their results.
 * @param {boolean} commit Whether or not to commit the transformations to the
 *   original root directory. If true then the changes are applied to the root
 *   directory, if false then the root directory will remain unchanged.
 * @param {fs-transform~Callback} executeCallback Callback to execute once the
 *   transformations have been completed or if an error has occurred.
 */
Transformer.prototype._execute = function (commit, executeCallback) {
  var self = this;
  this._fullDiff = "";

  async.series([
    // 0. Check to ensure we have all the required commands
    function checkRequiredCommands(cb) {
      self.driver.hasAllCommands(cb);
    },

    // 1. Create a working directory
    function setup(cb) {
      self.driver.createWorkingDirectory(cb);
    },

    // 2. Apply transformation rules
    function applyRules(cb) {
      async.mapSeries(self.rules, function (rule, ruleCallback) {
        self.applyRule(rule, ruleCallback);
      }, function (err) {
        cb(err, self);
      });
    },

    // 3. Fetch a diff between the working and the original root
    function fetchFullDiff(cb) {
      self.driver.workingDiff(function (err, diff) {
        if (err) { return cb(err); }
        fullDiffDebug(diff);
        self._fullDiff = diff;
        cb();
      });
    },

    // 4. Commit the changes if applicable, otherwise remove the working
    //    directory.
    function cleanup(cleanupCallback) {
      if (!commit) {
        return self.driver.removeWorkingDirectory(cleanupCallback);
      }

      var root = self.driver.root;
      var backup = root + '.bak';
      var working = self.driver.working;

      async.series([
        function backupRoot(cb) {
          self.driver.move(root, backup, cb);
        },
        function moveWorkingToRoot(cb) {
          self.driver.move(working, root, cb);
        },
        function removeBackup(cb) {
          self.driver.removeRecursive(backup, cb);
        }
      ], cleanupCallback);
    }
  ], function (err) {
    executeCallback(err, self);
  });
};

/**
 * Applys a given transformation rule.
 * @param {object} rule Rule to apply.
 * @param {function} cb Callback to execute once the rule has been processed.
 */
Transformer.prototype.applyRule = function (rule, cb) {
  if (!rule) {
    return cb(new Error('Null or undfined rule encountered.'));
  }

  if (!isString(rule.action)) {
    return cb(new Error('Invalid rule action: ' + rule.action));
  }

  var actionMethod = this.getAction(rule.action);
  if (!actionMethod) {
    return cb(new Error('Action not found: ' + rule.action));
  }

  this.pushResult(rule);
  actionMethod(rule, cb);
};

/**
 * Performs validations for copy and rename rules.
 * @param {object} rule Rule to validate.
 * @return {boolean} `true` if validations pass, `false` otherwise.
 */
Transformer.prototype._isValidRenameOrCopy = function (rule) {
  if (!isString(rule.source)) {
    this.addWarning(rule, 'Missing source file.');
    return false;
  }
  if (!isString(rule.dest)) {
    this.addWarning(rule, 'Missing destination file.');
    return false;
  }
  if (!this.driver.exists(rule.source)) {
    this.addWarning(rule, 'Source file does not exist.');
    return false;
  }
  if (this.driver.exists(rule.dest)) {
    this.addWarning(rule, 'Overwrites destination file.');
  }
  return true;
};

/**
 * Copy a file.
 * @param {object} rule Defines the source and destination for the file copy.
 * @param {function} cb Callback to execute once the file has been copied.
 */
Transformer.prototype.copy = function (rule, cb) {
  if (!this._isValidRenameOrCopy(rule)) {
    return cb();
  }
  var self = this;
  this.driver.copy(rule.source, rule.dest, function (err, output) {
    if (err) { return cb(err); }
    self.addNameChange(rule.source, rule.dest);
    cb();
  });
};

/**
 * Rename a file.
 * @param {object} rule Defines the source and destination for the file rename.
 * @param {function} cb Callback to execute once the file has been renamed.
 */
Transformer.prototype.rename = function (rule, cb) {
  if (!this._isValidRenameOrCopy(rule)) {
    return cb();
  }
  var self = this;
  this.driver.move(rule.source, rule.dest, function (err, output) {
    if (err) { return cb(err); }
    self.addNameChange(rule.source, rule.dest);
    cb();
  });
};

/**
 * Text search and replace for many files.
 * @param {object} rule Rules that define what to search for, replace, and
 *   exclude from the search.
 * @param {function} cb Callback to execute once the search and replace has been
 *   performed.
 */
Transformer.prototype.replace = function (rule, cb) {
  var self = this;

  var search = rule.search;
  if (!isString(search)) {
    this.addWarning(rule, 'Search pattern not specified.');
    return cb();
  }

  var replace = rule.replace;
  if (!isString(replace)) {
    this.addWarning(rule, 'Replacement not specified.');
    return cb();
  }

  var exclude = rule.exclude || [];
  if (!Array.isArray(exclude)) {
    this.addWarning(rule, 'Excludes not supplied as an array, omitting.');
    exclude = [];
  }

  this.driver.grep(rule.search, function (err, grepResult) {
    var files = [];

    // 1. Determine files & lines from grep
    grepResult.split('\n').forEach(function (line) {
      var match = line.match(/(.*):(\d+):.*/);
      if (match) {
        files.push({ name: match[1], line: match[2] });
      }
    });

    // 1.1 If no files were returned by grep, issue a warning and return
    if (files.length === 0) {
      self.addWarning(rule, 'Search did not return any results.');
      return cb();
    }

    // 2. Remove excluded files/lines from result set

    // 2.0 Filter out global excludes
    files = files.filter(function (file) {
      return !~self._globalExcludes.indexOf(file.name);
    });

    // 2.1 Keep track of when excludes are used to filter results
    var ruleApplied = [];
    exclude.forEach(function(excludeRule, index) {
      ruleApplied[index] = false;
    });

    // 2.2 Handle local rule file excludes
    files = files.filter(function (file) {
      var keepFile = true;
      exclude.forEach(function(name, index) {
{'fraction_non_alphanumeric': 0.1242797379430105, 'fraction_numerical': 0.001894387875917594, 'mean_word_length': 3.3307126986839855, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5856643', 'n_tokens_mistral': 7487, 'n_tokens_neox': 7345, 'n_words': 2471}