<commit_before>#!/usr/bin/env node

var fs = require('fs');
var nearley = require('../lib/nearley.js');
var inlineRequire = require('../lib/inline-require.js');
var nomnom = require('nomnom');

var opts = nomnom
	.script('nearleyc')
	.option('file', {
		position: 0,
		help: "An input .ne file (if not provided then read from stdin)",
	})
	.option('out', {
		abbr: 'o',
		help: "File to output to (defaults to stdout)",
	})
	.option('export', {
		abbr: 'e',
		help: "Variable to set the parser to",
		default: "module.exports"
	})
	.option('version', {
		abbr: '-v',
		flag: true,
		help: "Print version and exit",
		callback: function() {
			return require('../package.json').version;
		}
	})
	.parse();

function makeParser() {
    var language = require('../lib/nearley-language.js');

	return new nearley.Parser(language.rules, language.start);
}

function Compile(structure) {
	var un = 0;
	function unique() {
		return "  id" + (++un);
	}

	var outputRules = [];
	var outputNonterminals = [];

	function stringifyProductionRule(name, rule) {
		var tokenList = [];
		//console.log(name, rule);
		rule.tokens.forEach(function(token) {
			if (token.type && token.type === 'literal') {
				var str = token.data;
				if (str.length > 1) {
					var name = initNonterminal();
					var rules = str.split("").map(function(d) {
						return {type: 'literal', 'data':d};
					});
					var postprocessor = "function(d) {return d.join('');}";

					stringifyProductionRule(name, {tokens: rules, postprocessor: postprocessor});
					tokenList.push(name);
				} else if (str.length === 1) {
					tokenList.push(JSON.stringify(str));
				}
			} else if (typeof(token) === 'string') {
				if (token !== 'null') tokenList.push(nonterminalName(token));
            } else if (token instanceof RegExp) {
                tokenList.push(token.toString());
			} else {
                throw new Error("Should never get here");
            }
		})

		tokenList = "[" + tokenList.join(", ") + "]";
		var out = "rules.push(nearley.rule(" + name + ", " + tokenList +
			(rule.postprocessor ? ", " + rule.postprocessor : "") + "));";

		outputRules.push(out);
	}
	function initNonterminal(name) {
		var id = name || unique();
		var ref = nonterminalName(id);
		outputNonterminals.push(ref + " = {};");
		return ref;
	}
	function nonterminalName(name) {
		return "nonterminals['" + name + "']";
	}

	structure.forEach(function(productionRule) {
		var name = initNonterminal(productionRule.name);
		var rules = productionRule.rules;

		rules.forEach(function(rule) {
			stringifyProductionRule(name, rule);
		});
	});


	var output = "";
	var ws = "\n    ";

	output += "// Generated automatically by nearley.\n";
	output += opts.export + " = function() {";

	output += ws + "var nearley = " + inlineRequire(require, '../lib/nearley.js');
	output += ws + "var nonterminals = [];";
	output += ws + "var rules = [];";
	output += ws + "var id = function(a){return a[0];};";
	output += ws + outputNonterminals.join("\n    ");
	output += ws;
	output += ws + outputRules.join("\n    ");
	output += ws;
	output += ws + "return new nearley.Parser(rules, nonterminals['" + structure[0].name+ "']);";

	output += "\n};";<commit_msg>Update compiler to emit rule names as strings, literals as objects<commit_after>#!/usr/bin/env node

var fs = require('fs');
var nearley = require('../lib/nearley.js');
var inlineRequire = require('../lib/inline-require.js');
var nomnom = require('nomnom');

var opts = nomnom
	.script('nearleyc')
	.option('file', {
		position: 0,
		help: "An input .ne file (if not provided then read from stdin)",
	})
	.option('out', {
		abbr: 'o',
		help: "File to output to (defaults to stdout)",
	})
	.option('export', {
		abbr: 'e',
		help: "Variable to set the parser to",
		default: "module.exports"
	})
	.option('version', {
		abbr: '-v',
		flag: true,
		help: "Print version and exit",
		callback: function() {
			return require('../package.json').version;
		}
	})
	.parse();

function makeParser() {
    var language = require('../lib/nearley-language.js');

	return new nearley.Parser(language.rules, language.start);
}

function Compile(structure) {
	var un = 0;
	function unique() {
		return "  id" + (++un);
	}

	var outputRules = [];

	function stringifyProductionRule(name, rule) {
		var tokenList = [];

		rule.tokens.forEach(function(token) {
			if (token.type && token.type === 'literal') {
				var str = token.data;
				if (str.length > 1) {
					var rules = str.split("").map(function(d) {
						return {type: 'literal', 'data':d};
					});
					var postprocessor = "function(d) {return d.join('');}";

					stringifyProductionRule(name, {tokens: rules, postprocessor: postprocessor});
					tokenList.push(JSON.stringify({ literal: name }));
				} else if (str.length === 1) {
					tokenList.push(JSON.stringify({ literal: str}));
				}
			} else if (typeof(token) === 'string') {
				if (token !== 'null') tokenList.push(JSON.stringify(token));
            } else if (token instanceof RegExp) {
                tokenList.push(token.toString());
			} else {
                throw new Error("Should never get here");
            }
		})

		tokenList = "[" + tokenList.join(", ") + "]";
		var out = "rules.push(nearley.rule(" + JSON.stringify(name) + ", " + tokenList +
			(rule.postprocessor ? ", " + rule.postprocessor : "") + "));";

		outputRules.push(out);
	}

	structure.forEach(function(productionRule) {
		var rules = productionRule.rules;

		rules.forEach(function(rule) {
			stringifyProductionRule(productionRule.name, rule);
		});
	});


	var output = "";
	var ws = "\n    ";

	output += "// Generated automatically by nearley.\n";
	output += opts.export + " = function() {";

	output += ws + "var nearley = " + inlineRequire(require, '../lib/nearley.js');
	output += ws + "var rules = [];";
	output += ws + "var id = function(a){return a[0];};";
	output += ws;
	output += ws + outputRules.join("\n    ");
	output += ws;
	output += ws + "return new nearley.Parser(rules, " + JSON.stringify(structure[0].name) + ");";

	output += "\n};";
{'fraction_non_alphanumeric': 0.18998015873015872, 'fraction_numerical': 0.001984126984126984, 'mean_word_length': 3.3393113342898135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5954791', 'n_tokens_mistral': 2220, 'n_tokens_neox': 1972, 'n_words': 474}