<commit_before>var inlineRequire = require('../lib/inline-require.js');
var serialize = require('../lib/serialize.js');
var nearley = require('../lib/nearley.js');

function Compile(structure, opts) {
	var un = 0;
	function unique() {
		return "  id" + (++un);
	}

    function joiner(d) {
        return d.join('');
    }

	var outputRules = [];
    var body = [];

	function buildProcessedRule(name, rule) {
		var tokenList = [];

		rule.tokens.forEach(function(token) {
			if (token.type && token.type === 'literal') {
				var str = token.data;
				if (str.length > 1) {
					var rules = str.split("").map(function(d) {
						return { literal: d };
					});

					var newname = unique();
					buildProcessedRule(newname, {tokens: rules, postprocessor: joiner});
					tokenList.push(newname);
				} else if (str.length === 1) {
					tokenList.push({ literal: str });
				}
			} else if (typeof(token) === 'string') {
				if (token !== 'null') tokenList.push(token);
            } else if (token instanceof RegExp) {
                tokenList.push(token);
			} else {
                throw new Error("Should never get here");
            }
		});

		var out = nearley.rule(name, tokenList, rule.postprocessor);

		outputRules.push(out);
	}

	var firstName;
<commit_msg>Use Javascript "generated identifier" syntax for uniqids<commit_after>var inlineRequire = require('../lib/inline-require.js');
var serialize = require('../lib/serialize.js');
var nearley = require('../lib/nearley.js');

function Compile(structure, opts) {
	var un = 0;
	function unique(name) {
		return name + '$' + (++un);
	}

    function joiner(d) {
        return d.join('');
    }

	var outputRules = [];
    var body = [];

	function buildProcessedRule(name, rule) {
		var tokenList = [];

		rule.tokens.forEach(function(token) {
			if (token.type && token.type === 'literal') {
				var str = token.data;
				if (str.length > 1) {
					var rules = str.split("").map(function(d) {
						return { literal: d };
					});

					var newname = unique(name);
					buildProcessedRule(newname, {tokens: rules, postprocessor: joiner});
					tokenList.push(newname);
				} else if (str.length === 1) {
					tokenList.push({ literal: str });
				}
			} else if (typeof(token) === 'string') {
				if (token !== 'null') tokenList.push(token);
            } else if (token instanceof RegExp) {
                tokenList.push(token);
			} else {
                throw new Error("Should never get here");
            }
		});

		var out = nearley.rule(name, tokenList, rule.postprocessor);

		outputRules.push(out);
	}

	var firstName;

{'fraction_non_alphanumeric': 0.1658310836868492, 'fraction_numerical': 0.0023139220979560356, 'mean_word_length': 2.7110157367668095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2075768', 'n_tokens_mistral': 957, 'n_tokens_neox': 818, 'n_words': 196}