<commit_before>(function () {
if (typeof(window) === 'undefined') {
    var nearley = require('../lib/nearley.js');
} else {
    var nearley = window.nearley;
}

function Compile(structure, opts) {
    var unique = uniquer();

    var result = {
        rules: [],
        body: [], // @directives list
        start: ''
    };

    for (var i = 0; i < structure.length; i++) {
        var productionRule = structure[i];
        if (productionRule.body) {
            // This isn't a rule, it's an @directive.
            if (!opts.nojs) {
                result.body.push(productionRule.body);
            }
        } else {
            produceRules(productionRule.name, productionRule.rules);
            if (!result.start) {
                result.start = productionRule.name;
            }
        }
    }

    return result;

    function produceRules(name, rules) {
        for (var i = 0; i < rules.length; i++) {
            var rule = rules[i];
            buildRule(name, rule);
        }
    }

    function buildRule(ruleName, rule) {
        var tokens = [];
        for (var i = 0; i < rule.tokens.length; i++) {
            var token = buildToken(ruleName, rule.tokens[i]);
            if (token) {
                tokens.push(token);
            }
        }
        var out = new nearley.Rule(
            ruleName,
            tokens,
            (opts.nojs ? null : rule.postprocess)
        );
        result.rules.push(out);
        return out;
    }

    function buildToken(ruleName, token) {
        if (token.literal) {
            var str = token.literal;
            if (str.length > 1) {
                var rules = str.split("").map(function(d) {
                    return {
                        literal: d
                    };
                });

                var newname = unique(ruleName + "$string");<commit_msg>lib/compile: make buildRule (locally) functional<commit_after>(function () {
if (typeof(window) === 'undefined') {
    var nearley = require('../lib/nearley.js');
} else {
    var nearley = window.nearley;
}

function Compile(structure, opts) {
    var unique = uniquer();

    var result = {
        rules: [],
        body: [], // @directives list
        start: ''
    };

    for (var i = 0; i < structure.length; i++) {
        var productionRule = structure[i];
        if (productionRule.body) {
            // This isn't a rule, it's an @directive.
            if (!opts.nojs) {
                result.body.push(productionRule.body);
            }
        } else {
            produceRules(productionRule.name, productionRule.rules);
            if (!result.start) {
                result.start = productionRule.name;
            }
        }
    }

    return result;

    function produceRules(name, rules) {
        for (var i = 0; i < rules.length; i++) {
            var rule = buildRule(name, rules[i]);
            if (opts.nojs) {
                rule.postprocess = null;
            }
            result.rules.push(rule);
        }
    }

    function buildRule(ruleName, rule) {
        var tokens = [];
        for (var i = 0; i < rule.tokens.length; i++) {
            var token = buildToken(ruleName, rule.tokens[i]);
            if (token) {
                tokens.push(token);
            }
        }
        return new nearley.Rule(
            ruleName,
            tokens,
            rule.postprocess
        );
    }

    function buildToken(ruleName, token) {
        if (token.literal) {
            var str = token.literal;
            if (str.length > 1) {
                var rules = str.split("").map(function(d) {
                    return {
                        literal: d
                    };
                });

                var newname = unique(ruleName + "$string");
{'fraction_non_alphanumeric': 0.12838196286472148, 'fraction_numerical': 0.002122015915119363, 'mean_word_length': 1.3942857142857144, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6763599', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1035, 'n_words': 245}