<commit_before>// --------------------------------------------------------------------
// Imports
// --------------------------------------------------------------------

var common = require('./common.js');
var errors = require('./errors.js');
var InputStream = require('./InputStream.js');
var Interval = require('./Interval.js');
var Node = require('./Node.js');
var pexprs = require('./pexprs.js');
var attributes = require('./attributes.js');

var awlib = require('awlib');
var keysDo = awlib.objectUtils.keysDo;
var valuesDo = awlib.objectUtils.valuesDo;
var formals = awlib.objectUtils.formals;
var makeStringBuffer = awlib.objectUtils.stringBuffer;
var printString = awlib.stringUtils.printString;
var equals = awlib.equals.equals;

// --------------------------------------------------------------------
// Private stuff
// --------------------------------------------------------------------

function Grammar(name, superGrammar, ruleDecls, ruleDict, optNamespace) {
  // N.B. Consider compareGrammars() in the test code when adding instvars.
  this.namespaceName = optNamespace ? optNamespace.name : undefined;
  this.name = name;
  this.superGrammar = superGrammar;
  this.ruleDecls = ruleDecls;
  this.ruleDict = ruleDict;
  this.constructors = this.ctors = this.createConstructors();

  if (optNamespace) {
    optNamespace.install(this.name, this);
  }
}

Grammar.prototype = {
  ruleDict: {
    _: pexprs.anything,
    end: new pexprs.Not(pexprs.anything),
    fail: pexprs.fail,
    space: pexprs.makePrim(/[\s]/).withDescription('space'),
    alnum: pexprs.makePrim(/[0-9a-zA-Z]/).withDescription('alpha-numeric character'),
    letter: pexprs.makePrim(/[a-zA-Z]/).withDescription('letter'),
    lower: pexprs.makePrim(/[a-z]/).withDescription('lower-case letter'),
    upper: pexprs.makePrim(/[A-Z]/).withDescription('upper-case letter'),
    digit: pexprs.makePrim(/[0-9]/).withDescription('digit'),
    hexDigit: pexprs.makePrim(/[0-9a-fA-F]/).withDescription('hexadecimal digit')
  },

  construct: function(ruleName, args) {
    var body = this.ruleDict[ruleName];
    if (!body || !body.check(this, args) || args.length !== body.getArity()) {
      throw new errors.InvalidConstructorCall(this, ruleName, args);
    }
    var interval = new Interval(InputStream.newFor(args), 0, args.length);
    return new Node(this, ruleName, args, interval);
  },

  createConstructors: function() {
    var self = this;
    var constructors = {};

    for (var ruleName in this.ruleDict) {
      // We want *all* properties, not just own properties, because of
      // supergrammars.

      // also WOW I can't believe I was bitten AGAIN by Javascript's
      // silly mutable for-bound variables
      (function(ruleName) {
	constructors[ruleName] = function(/* val1, val2, ... */) {
	  return self.construct(ruleName, Array.prototype.slice.call(arguments));
	};
      })(ruleName);
    }
    return constructors;
  },

  match: function(obj, startRule, optThrowOnFail) {
    return this.matchContents([obj], startRule, optThrowOnFail);
  },

  matchContents: function(obj, startRule, optThrowOnFail) {
    var inputStream = InputStream.newFor(obj);
    var bindings = [];
    var succeeded = new pexprs.Apply(startRule).eval(optThrowOnFail, undefined, this, inputStream, bindings);

    if (succeeded) {
      // This match only succeeded if the start rule consumed all of the input.
      if (common.isSyntactic(startRule)) {
        this.skipSpaces(inputStream);
      }
      succeeded = pexprs.end.eval(optThrowOnFail, false, this, inputStream, []);
    }

    if (succeeded) {
      var node = bindings[0];
      var stack = [undefined];
      function setParentAction() {
        stack.push(this);
        this.args.forEach(function(arg) { setParents(arg); });
        stack.pop();
        this.parent = stack[stack.length - 1];
      };
      var setParents = this.synthesizedAttribute({
        _terminal: function() { this.parent = stack[stack.length - 1]; },
        _list: setParentAction,
        _default: setParentAction
      });
      setParents(node);
      return node;
    } else if (optThrowOnFail) {
      throw new errors.MatchFailure(inputStream, this.ruleDict);
    } else {
      return false;
    }
  },

  skipSpaces: function(inputStream) {
    while (true) {
      var origPos = inputStream.pos;
      if (!this.ruleDict.space.eval(false, false, this, inputStream, [])) {
        inputStream.pos = origPos;
        break;
      }<commit_msg>made setParents a semanticAction rather than a synthesizedAttribute<commit_after>// --------------------------------------------------------------------
// Imports
// --------------------------------------------------------------------

var common = require('./common.js');
var errors = require('./errors.js');
var InputStream = require('./InputStream.js');
var Interval = require('./Interval.js');
var Node = require('./Node.js');
var pexprs = require('./pexprs.js');
var attributes = require('./attributes.js');

var awlib = require('awlib');
var keysDo = awlib.objectUtils.keysDo;
var valuesDo = awlib.objectUtils.valuesDo;
var formals = awlib.objectUtils.formals;
var makeStringBuffer = awlib.objectUtils.stringBuffer;
var printString = awlib.stringUtils.printString;
var equals = awlib.equals.equals;

// --------------------------------------------------------------------
// Private stuff
// --------------------------------------------------------------------

function Grammar(name, superGrammar, ruleDecls, ruleDict, optNamespace) {
  // N.B. Consider compareGrammars() in the test code when adding instvars.
  this.namespaceName = optNamespace ? optNamespace.name : undefined;
  this.name = name;
  this.superGrammar = superGrammar;
  this.ruleDecls = ruleDecls;
  this.ruleDict = ruleDict;
  this.constructors = this.ctors = this.createConstructors();

  if (optNamespace) {
    optNamespace.install(this.name, this);
  }
}

Grammar.prototype = {
  ruleDict: {
    _: pexprs.anything,
    end: new pexprs.Not(pexprs.anything),
    fail: pexprs.fail,
    space: pexprs.makePrim(/[\s]/).withDescription('space'),
    alnum: pexprs.makePrim(/[0-9a-zA-Z]/).withDescription('alpha-numeric character'),
    letter: pexprs.makePrim(/[a-zA-Z]/).withDescription('letter'),
    lower: pexprs.makePrim(/[a-z]/).withDescription('lower-case letter'),
    upper: pexprs.makePrim(/[A-Z]/).withDescription('upper-case letter'),
    digit: pexprs.makePrim(/[0-9]/).withDescription('digit'),
    hexDigit: pexprs.makePrim(/[0-9a-fA-F]/).withDescription('hexadecimal digit')
  },

  construct: function(ruleName, args) {
    var body = this.ruleDict[ruleName];
    if (!body || !body.check(this, args) || args.length !== body.getArity()) {
      throw new errors.InvalidConstructorCall(this, ruleName, args);
    }
    var interval = new Interval(InputStream.newFor(args), 0, args.length);
    return new Node(this, ruleName, args, interval);
  },

  createConstructors: function() {
    var self = this;
    var constructors = {};

    for (var ruleName in this.ruleDict) {
      // We want *all* properties, not just own properties, because of
      // supergrammars.

      // also WOW I can't believe I was bitten AGAIN by Javascript's
      // silly mutable for-bound variables
      (function(ruleName) {
	constructors[ruleName] = function(/* val1, val2, ... */) {
	  return self.construct(ruleName, Array.prototype.slice.call(arguments));
	};
      })(ruleName);
    }
    return constructors;
  },

  match: function(obj, startRule, optThrowOnFail) {
    return this.matchContents([obj], startRule, optThrowOnFail);
  },

  matchContents: function(obj, startRule, optThrowOnFail) {
    var inputStream = InputStream.newFor(obj);
    var bindings = [];
    var succeeded = new pexprs.Apply(startRule).eval(optThrowOnFail, undefined, this, inputStream, bindings);

    if (succeeded) {
      // This match only succeeded if the start rule consumed all of the input.
      if (common.isSyntactic(startRule)) {
        this.skipSpaces(inputStream);
      }
      succeeded = pexprs.end.eval(optThrowOnFail, false, this, inputStream, []);
    }

    if (succeeded) {
      var node = bindings[0];
      var stack = [undefined];
      function setParentAction() {
        stack.push(this);
        this.args.forEach(function(arg) { setParents(arg); });
        stack.pop();
        this.parent = stack[stack.length - 1];
      };
      var setParents = this.semanticAction({
        _terminal: function() { this.parent = stack[stack.length - 1]; },
        _list: setParentAction,
        _default: setParentAction
      });
      setParents(node);
      return node;
    } else if (optThrowOnFail) {
      throw new errors.MatchFailure(inputStream, this.ruleDict);
    } else {
      return false;
    }
  },

  skipSpaces: function(inputStream) {
    while (true) {
      var origPos = inputStream.pos;
      if (!this.ruleDict.space.eval(false, false, this, inputStream, [])) {
        inputStream.pos = origPos;
        break;
      }
{'fraction_non_alphanumeric': 0.19427811044577512, 'fraction_numerical': 0.0026613439787092482, 'mean_word_length': 2.6018370607028753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6046508', 'n_tokens_mistral': 2725, 'n_tokens_neox': 2672, 'n_words': 587}