<commit_before>(function () {
if (typeof(window) === 'undefined') {
    var nearley = require('../lib/nearley.js');
} else {
    var nearley = window.nearley;
}
var Compile = function (structure, opts) {
    var un = 0;

    function unique(name) {
        return name + '$' + (++un);
    }

    function joiner(d) {
        return d.join('');
    }

    var outputRules = [];
    var body = []; // @directives list

    function buildProcessedRule(name, rule) {
        var tokenList = [];

        rule.tokens.forEach(function(token) {
            if (token.literal) {
                var str = token.literal;
                if (str.length > 1) {
                    var rules = str.split("").map(function(d) {
                        return {
                            literal: d
                        };
                    });

                    var newname = unique(" string");
                    buildProcessedRule(newname, {
                        tokens: rules,
                        postprocess: joiner
                    });
                    tokenList.push(newname);
                } else if (str.length === 1) {
                    tokenList.push({
                        literal: str
                    });
                }
            } else if (token.subexpression) {
                var data = token.subexpression;
                var name = unique(" subexpression");
                structure.push({"name": name, "rules": data});
                tokenList.push(name);
            } else if (token.ebnf) {
                var data = token.ebnf;
                var modifier = token.modifier;
                var name = unique(" ebnf");
                var arrconcat = (function(d) {
                    return [d[0]].concat(d[1]);
                }).toString();

                switch (modifier) {
                    case ":+":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [data],
                            }, {
                                "tokens": [data, name],
                                "postprocess": arrconcat
                            }]
                        });
                        tokenList.push(name);
                        break;
                    case ":*":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [],
                            }, {
                                "tokens": [data, name],
                                "postprocess": arrconcat
                            }]
                        });
                        tokenList.push(name);
                        break;
                    case ":?":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [data],
                                "postprocess": "id"
                            }, {
                                "tokens": [],
                                "postprocess": "function(d) {return null;}"
                            }]
                        });
                        tokenList.push(name);
                        break;
                }
            } else if (typeof(token) === 'string') {
                if (token !== 'null') tokenList.push(token);
            } else if (token instanceof RegExp) {
                tokenList.push(token);
            } else {
                throw new Error("Should never get here");
            }
        });

        var out = new nearley.Rule(
            name,
            tokenList,
            (opts.nojs ? null : rule.postprocess)
        );

        outputRules.push(out);
    }

    var firstName;

    var i=0;
    while (i<structure.length) {
        var productionRule = structure[i];
        if (productionRule.body) {
            // This isn't a rule, it's an @directive.
            if (!opts.nojs) body.push(productionRule.body);
        } else {
            var rules = productionRule.rules;
            if (!firstName) firstName = productionRule.name;
            rules.forEach(function(rule) {
                buildProcessedRule(productionRule.name, rule);
            });
        }
        i++;
    }

    return {
        rules: outputRules,
        body: body,
        start: firstName
    };
}

if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
   module.exports = Compile;
} else {
   window.Compile = Compile;
}
})();
<commit_msg>lib/compile: un-shadow rule name in token handling context<commit_after>(function () {
if (typeof(window) === 'undefined') {
    var nearley = require('../lib/nearley.js');
} else {
    var nearley = window.nearley;
}
var Compile = function (structure, opts) {
    var un = 0;

    function unique(name) {
        return name + '$' + (++un);
    }

    function joiner(d) {
        return d.join('');
    }

    var outputRules = [];
    var body = []; // @directives list

    function buildProcessedRule(ruleName, rule) {
        var tokenList = [];

        rule.tokens.forEach(function(token) {
            if (token.literal) {
                var str = token.literal;
                if (str.length > 1) {
                    var rules = str.split("").map(function(d) {
                        return {
                            literal: d
                        };
                    });

                    var newname = unique(" string");
                    buildProcessedRule(newname, {
                        tokens: rules,
                        postprocess: joiner
                    });
                    tokenList.push(newname);
                } else if (str.length === 1) {
                    tokenList.push({
                        literal: str
                    });
                }
            } else if (token.subexpression) {
                var data = token.subexpression;
                var name = unique(" subexpression");
                structure.push({"name": name, "rules": data});
                tokenList.push(name);
            } else if (token.ebnf) {
                var data = token.ebnf;
                var modifier = token.modifier;
                var name = unique(" ebnf");
                var arrconcat = (function(d) {
                    return [d[0]].concat(d[1]);
                }).toString();

                switch (modifier) {
                    case ":+":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [data],
                            }, {
                                "tokens": [data, name],
                                "postprocess": arrconcat
                            }]
                        });
                        tokenList.push(name);
                        break;
                    case ":*":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [],
                            }, {
                                "tokens": [data, name],
                                "postprocess": arrconcat
                            }]
                        });
                        tokenList.push(name);
                        break;
                    case ":?":
                        structure.push({
                            "name": name,
                            "rules": [{
                                "tokens": [data],
                                "postprocess": "id"
                            }, {
                                "tokens": [],
                                "postprocess": "function(d) {return null;}"
                            }]
                        });
                        tokenList.push(name);
                        break;
                }
            } else if (typeof(token) === 'string') {
                if (token !== 'null') tokenList.push(token);
            } else if (token instanceof RegExp) {
                tokenList.push(token);
            } else {
                throw new Error("Should never get here");
            }
        });

        var out = new nearley.Rule(
            ruleName,
            tokenList,
            (opts.nojs ? null : rule.postprocess)
        );

        outputRules.push(out);
    }

    var firstName;

    var i=0;
    while (i<structure.length) {
        var productionRule = structure[i];
        if (productionRule.body) {
            // This isn't a rule, it's an @directive.
            if (!opts.nojs) body.push(productionRule.body);
        } else {
            var rules = productionRule.rules;
            if (!firstName) firstName = productionRule.name;
            rules.forEach(function(rule) {
                buildProcessedRule(productionRule.name, rule);
            });
        }
        i++;
    }

    return {
        rules: outputRules,
        body: body,
        start: firstName
    };
}

if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
   module.exports = Compile;
} else {
   window.Compile = Compile;
}
})();

{'fraction_non_alphanumeric': 0.11498555995293615, 'fraction_numerical': 0.0012835597390095197, 'mean_word_length': 0.9050529747351264, 'pattern_counts': {'":': 48, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6109041', 'n_tokens_mistral': 2285, 'n_tokens_neox': 2124, 'n_words': 473}