<commit_before>var lisp = require('./lisp');

var uuid = (function() {
  function s4() {
    return Math.floor((1 + Math.random()) * 0x10000)
      .toString(16)
      .substring(1);
  }

  return function() {
    return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
      s4() + '-' + s4() + s4() + s4();
  }
})();

const EOF = uuid();

function makeRules(ast) {
  var table = [];
  var regexTable = [];

  ast.value.forEach(function(node) {
    var list = node.value;
    var lhs = list[0].value;
    for (var i = 1; i < list.length; i += 2) {
      var tokenList = list[i].value;
      var rhs = [];
      if (tokenList.length === 0) throw new Error("Can't handle empty for now...");

      tokenList.forEach(function(elem) {
        if (elem.type === 'list' || elem.type === 'string') {
          var string = elem, priority = 0;
          if (elem.type === 'list'){
            var string = elem.value[0];
            priority = elem.value[1] ? elem.value[1].value : 0;
          }

          var maybeMatchOnEmpty = ''.match(string.value);
          if (maybeMatchOnEmpty && maybeMatchOnEmpty.index === 0) {
            throw new Error("Can you not match on emtpy string plz with `" + string.value + "`");
          }

          var regex = new RegExp(string.value);
          var id = uuid();
          var maybeRegex = regexTable.filter(r => r.rhs.toString() === regex.toString())[0];
          if (maybeRegex) {
            id = maybeRegex.lhs;
          } else {
            regexTable.push({
              lhs: id,
              rhs: regex,
              priority: priority,
              lambda: null,
            });
          }
          rhs.push(id);
        } else if (elem.type === 'identifier'){
          rhs.push(elem.value);
        }
      });
      var lambda = list[i + 1];
      lambda.isMacro = true;
      table.push({
        lhs: lhs,
        rhs: rhs,
        lambda: lambda,
      });
    }
  });

  regexTable.push({
    lhs: EOF,
    rhs: new RegExp(/$/),<commit_msg>Add error message when receiving an unexpected node type when turning the CFG into a rule table.<commit_after>var lisp = require('./lisp');

var uuid = (function() {
  function s4() {
    return Math.floor((1 + Math.random()) * 0x10000)
      .toString(16)
      .substring(1);
  }

  return function() {
    return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
      s4() + '-' + s4() + s4() + s4();
  }
})();

const EOF = uuid();

function makeRules(ast) {
  var table = [];
  var regexTable = [];

  ast.value.forEach(function(node) {
    var list = node.value;
    var lhs = list[0].value;
    for (var i = 1; i < list.length; i += 2) {
      var tokenList = list[i].value;
      var rhs = [];
      if (tokenList.length === 0) throw new Error("Can't handle empty for now...");

      tokenList.forEach(function(elem, j) {
        if (elem.type === 'list' || elem.type === 'string') {
          var string = elem, priority = 0;
          if (elem.type === 'list'){
            var string = elem.value[0];
            priority = elem.value[1] ? elem.value[1].value : 0;
          }

          var maybeMatchOnEmpty = ''.match(string.value);
          if (maybeMatchOnEmpty && maybeMatchOnEmpty.index === 0) {
            throw new Error("Can you not match on emtpy string plz with `" + string.value + "`");
          }

          var regex = new RegExp(string.value);
          var id = uuid();
          var maybeRegex = regexTable.filter(r => r.rhs.toString() === regex.toString())[0];
          if (maybeRegex) {
            id = maybeRegex.lhs;
          } else {
            regexTable.push({
              lhs: id,
              rhs: regex,
              priority: priority,
              lambda: null,
            });
          }
          rhs.push(id);
        } else if (elem.type === 'identifier'){
          rhs.push(elem.value);
        } else {
          throw new Error("Unexpected AST node of type `" + elem.type + "` found at position inside rule #" + i + " at position " + j);
        }
      });
      var lambda = list[i + 1];
      lambda.isMacro = true;
      table.push({
        lhs: lhs,
        rhs: rhs,
        lambda: lambda,
      });
    }
  });

  regexTable.push({
    lhs: EOF,
    rhs: new RegExp(/$/),
{'fraction_non_alphanumeric': 0.1660769049304081, 'fraction_numerical': 0.014626091059212079, 'mean_word_length': 1.7604166666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4494910', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1349, 'n_words': 344}