<commit_before>"use strict";

var TextQuoter = require("./text-quoter");
var TextUtil = require("./text-util");
var TextGraph = require("./graph");

var defaultOptions = {
  hiddenPaths: [],
  useColor: true,
  maxSourceLines: 6,
  parent: null,
  showSource: true,
  showTrace: false,
  showFullPath: false,
  maxPathLength: 64,
  matchesNode: function(_node, _options = {}) { return true; },
};

var VLINE_STYLES = [
  { color: "yellow" },
  { color: "magenta" },
  { color: "blue" },
  { color: "white" },
  { color: "green" }
];

var Tracer = function(source, opt) {
  this.options = {};

  for (var key in defaultOptions) {
    this.options[key] =
      opt && opt[key] !== undefined ? opt[key] : defaultOptions[key];
  }

  this.parent = this.options.parent;
  this.quoter = new TextQuoter(source, { useColor: this.options.useColor });
  this.hiddenPatterns = [];

  for (var i = 0; i < this.options.hiddenPaths.length; i++) {
    var pattern = this.options.hiddenPaths[i];

    if (pattern instanceof RegExp) {
      this.hiddenPatterns[i] = pattern;
    } else {
      this.hiddenPatterns[i] = new RegExp("(^|/)" + pattern + "(/|$)");
    }
  }

  this.headStringMap = {
    "rule.enter": this.setTextStyle("+ ", { color: "cyan" }),
    "rule.match": this.setTextStyle("o ", { color: "green" }),
    "rule.fail": this.setTextStyle("x ", { color: "red" }),
    error: this.setTextStyle("! ", { color: "red" })
  };

  this.typeStringMap = {
    "rule.enter": this.setTextStyle("ENTER", { color: "cyan" }),
    "rule.match": this.setTextStyle("MATCH", { color: "green" }),
    "rule.fail": this.setTextStyle("FAIL ", { color: "red" }),
    error: this.setTextStyle("ERROR", { color: "red" })
  };

  this.init();
};

Tracer.prototype.init = function() {
  this.root = {
    type: "root",
    path: "",
    parent: null,
    matches: [],
    fails: [],
    rule: "",
    location: {
      start: { offset: 0, line: 0, column: 0 },
      end: { offset: 0, line: 0, column: 0 }
    }
  };

  this.currentNode = this.root;
  this.maxFailPos = 0;
  this.maxFails = [];
  this.currentLevel = 0;
  this.numNodes = 0;
};

Tracer.prototype.setTextStyle = function(str, style) {
  return this.quoter.setTextStyle(str, style);
};

var _convertToZeroBasedLocation = function(location) {
  return {
    start: {
      offset: location.start.offset,
      line: location.start.line - 1,
      column: location.start.column - 1
    },
    end: {
      offset: location.end.offset,
      line: location.end.line - 1,
      column: location.end.column - 1
    }
  };
};

Tracer.prototype.getSourceLines = function(quoteString, location, maxLines) {
  var location = _convertToZeroBasedLocation(location);

  return this.quoter.getQuotedLines(
    quoteString,
    location.start.line,
    location.start.column,
    location.end.line,
    location.end.column,
    maxLines
  );
};

Tracer.prototype.isHidden = function(node) {
  var path = node.path + node.rule;

  for (var i = 0; i < this.hiddenPatterns.length; i++) {
    var pattern = this.hiddenPatterns[i];

    if (pattern.test(path)) {
      return true;
    }
  }

  return false;
};

Tracer.prototype.trace = function(evt) {
  if (this.parent && this.parent.trace) {
    this.parent.trace(evt);
  }

  switch (evt.type) {
    case "rule.enter":
      this.onEnter(evt);
      break;
    case "rule.match":
      this.onMatch(evt);
      break;
    default:
      this.onFail(evt);
      break;
  }
};

Tracer.prototype.printNode = function(level, node) {
  if (this.isHidden(node)) return;

  var lines = this.buildNodeText(node, this.options.showSource, " ");
  var style = VLINE_STYLES[level % VLINE_STYLES.length];
  var tailIndent = TextUtil.makeIndent(level + 1);
  var headIndent =
    TextUtil.makeIndent(level) + this.typeStringMap[node.type] + " ";

  lines = lines.map(function(e, i) {
    if (i == 0) {
      return headIndent + e;
    } else {
      return tailIndent + e;
    }
  });

  console.log(lines.join("\n"));<commit_msg>add an option to set an output function<commit_after>"use strict";

var TextQuoter = require("./text-quoter");
var TextUtil = require("./text-util");
var TextGraph = require("./graph");

var defaultOptions = {
  hiddenPaths: [],
  useColor: true,
  maxSourceLines: 6,
  parent: null,
  showSource: true,
  showTrace: false,
  showFullPath: false,
  maxPathLength: 64,
  matchesNode: function(_node, _options = {}) { return true; },
  output: console.log,
};

var VLINE_STYLES = [
  { color: "yellow" },
  { color: "magenta" },
  { color: "blue" },
  { color: "white" },
  { color: "green" }
];

var Tracer = function(source, opt) {
  this.options = {};

  for (var key in defaultOptions) {
    this.options[key] =
      opt && opt[key] !== undefined ? opt[key] : defaultOptions[key];
  }

  this.parent = this.options.parent;
  this.quoter = new TextQuoter(source, { useColor: this.options.useColor });
  this.hiddenPatterns = [];

  for (var i = 0; i < this.options.hiddenPaths.length; i++) {
    var pattern = this.options.hiddenPaths[i];

    if (pattern instanceof RegExp) {
      this.hiddenPatterns[i] = pattern;
    } else {
      this.hiddenPatterns[i] = new RegExp("(^|/)" + pattern + "(/|$)");
    }
  }

  this.headStringMap = {
    "rule.enter": this.setTextStyle("+ ", { color: "cyan" }),
    "rule.match": this.setTextStyle("o ", { color: "green" }),
    "rule.fail": this.setTextStyle("x ", { color: "red" }),
    error: this.setTextStyle("! ", { color: "red" })
  };

  this.typeStringMap = {
    "rule.enter": this.setTextStyle("ENTER", { color: "cyan" }),
    "rule.match": this.setTextStyle("MATCH", { color: "green" }),
    "rule.fail": this.setTextStyle("FAIL ", { color: "red" }),
    error: this.setTextStyle("ERROR", { color: "red" })
  };

  this.init();
};

Tracer.prototype.init = function() {
  this.root = {
    type: "root",
    path: "",
    parent: null,
    matches: [],
    fails: [],
    rule: "",
    location: {
      start: { offset: 0, line: 0, column: 0 },
      end: { offset: 0, line: 0, column: 0 }
    }
  };

  this.currentNode = this.root;
  this.maxFailPos = 0;
  this.maxFails = [];
  this.currentLevel = 0;
  this.numNodes = 0;
};

Tracer.prototype.setTextStyle = function(str, style) {
  return this.quoter.setTextStyle(str, style);
};

var _convertToZeroBasedLocation = function(location) {
  return {
    start: {
      offset: location.start.offset,
      line: location.start.line - 1,
      column: location.start.column - 1
    },
    end: {
      offset: location.end.offset,
      line: location.end.line - 1,
      column: location.end.column - 1
    }
  };
};

Tracer.prototype.getSourceLines = function(quoteString, location, maxLines) {
  var location = _convertToZeroBasedLocation(location);

  return this.quoter.getQuotedLines(
    quoteString,
    location.start.line,
    location.start.column,
    location.end.line,
    location.end.column,
    maxLines
  );
};

Tracer.prototype.isHidden = function(node) {
  var path = node.path + node.rule;

  for (var i = 0; i < this.hiddenPatterns.length; i++) {
    var pattern = this.hiddenPatterns[i];

    if (pattern.test(path)) {
      return true;
    }
  }

  return false;
};

Tracer.prototype.trace = function(evt) {
  if (this.parent && this.parent.trace) {
    this.parent.trace(evt);
  }

  switch (evt.type) {
    case "rule.enter":
      this.onEnter(evt);
      break;
    case "rule.match":
      this.onMatch(evt);
      break;
    default:
      this.onFail(evt);
      break;
  }
};

Tracer.prototype.printNode = function(level, node) {
  if (this.isHidden(node)) return;

  var lines = this.buildNodeText(node, this.options.showSource, " ");
  var style = VLINE_STYLES[level % VLINE_STYLES.length];
  var tailIndent = TextUtil.makeIndent(level + 1);
  var headIndent =
    TextUtil.makeIndent(level) + this.typeStringMap[node.type] + " ";

  lines = lines.map(function(e, i) {
    if (i == 0) {
      return headIndent + e;
    } else {
      return tailIndent + e;
    }
  });

  this.options.output(lines.join("\n"));
{'fraction_non_alphanumeric': 0.16842890906829208, 'fraction_numerical': 0.004975743251648215, 'mean_word_length': 3.007976071784646, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2581884', 'n_tokens_mistral': 2844, 'n_tokens_neox': 2797, 'n_words': 568}