<commit_before>"use strict"; "use restrict";

var require = require || function(f) { load(f); };
require.paths && typeof __dirname !== "undefined" && require.paths.unshift(__dirname);
/* global Narcissus, tkn */ (typeof Narcissus === "undefined") && require("narcissus.js");
var Fmt = Fmt || require("fmt.js") || Fmt;
var Ref = Ref || require("ref.js") || Ref;
var Log = Log || require("log.js") || Log;
var Assert = Assert || require("assert.js") || Assert;
var Comments = Comments || require("comments.js") || Comments;
var log = (typeof console !== "undefined") && console.log || print;

var Shaper = (function() {
    Array.isArray = Array.isArray || function(o) {
        return Object.prototype.toString.call(o) === "[object Array]";
    };
    function error(node, msg) {
        log(Fmt("{0}:{1} error: {2}", node.tokenizer.filename, node.lineno, msg));
        (typeof quit === "undefined" ? process.exit : quit)(-1);
    }
    function deprecated(obj, params) {
        obj[params.was] = function() {
            throw new Error(Fmt.obj((params.now ?
                                     "{was} is deprecated since version {since}, use {now} instead" :
                                     "{was} is deprecated since version {since}"), params));
        };
    }

    var traverseData = (function() {
        var o = [];

        o[tkn.ARRAY_COMP] = [/*expr*/"expression", /*COMP_TAIL*/"tail"];
        o[tkn.CASE] = [/*expr*/"caseLabel", /*BLOCK*/"statements"];
        o[tkn.CATCH] = [/*IDENTIFIER*/"_name", /*expr*/"guard", /*BLOCK*/"block"];
        o[tkn.COMP_TAIL] = [/*[FOR_IN]*/"children", /*expr*/"guard"]; // has children but not last
        o[tkn.DEFAULT] = [/*BLOCK*/"statements"];
        o[tkn.DO] = [/*stmt*/"body", /*expr*/"condition"];
        o[tkn.FOR] = [/*expr*/"setup", /*expr*/"condition", /*expr*/"update", /*stmt*/"body"];
        o[tkn.FOR_IN] = [/*IDENTIFIER|VAR*/"_iterator", /*expr*/"object", /*stmt*/"body"];
        o[tkn.FUNCTION] = [/*IDENTIFIER*/"_name", /*[IDENTIFIER]*/"_params", /*SCRIPT*/"body"];
        o[tkn.GENERATOR] = [/*expr*/"expression", /*COMP_TAIL*/"tail"];
        o[tkn.GETTER] = [/*SCRIPT*/"body"];
        o[tkn.IF] = [/*expr*/"condition", /*stmt*/"thenPart", /*stmt*/"elsePart"];
        o[tkn.LABEL] = [/*stmt*/"statement"]; // TODO should /*IDENTIFIER*/"_label" be captured?
        o[tkn.LET_BLOCK] = [/*LET*/"variables", /*expr*/"expression", /*BLOCK*/"block"];
        o[tkn.RETURN] = [/*expr*/"value"];
        o[tkn.SEMICOLON] = [/*expr*/"expression"];
        o[tkn.SETTER] = [/*SCRIPT*/"body"];
        o[tkn.SWITCH] = [/*expr*/"discriminant", /*[CASE|DEFAULT]*/"cases"];
        o[tkn.THROW] = [/*expr*/"exception"];
        o[tkn.TRY] = [/*BLOCK*/"tryBlock", /*[CATCH]*/"catchClauses", /*BLOCK*/"finallyBlock"];
        o[tkn.WHILE] = [/*expr*/"condition", /*stmt*/"body"];
        o[tkn.WITH] = [/*expr*/"object", /*stmt*/"body"];
        o[tkn.YIELD] = [/*expr*/"value"];

/*
        // TODO these properties aren't nodes but may still be relevant
        x[tkn.ASSIGN] = ["assignOp"]; // number ("value" has string representation)
        x[tkn.INCREMENT] = ["postfix"]; // boolean ("value" is just "++")
        x[tkn.DECREMENT] = ["postfix"]; // boolean
        x[tkn.FUNCTION] = ["functionForm"]; // number
        x[tkn.FOR_IN] = ["isEach"]; // boolean
        x[tkn.SWITCH] = ["defaultIndex"]; // number
        x[tkn.LABEL] = ["label"]; // string
        x[tkn.IDENTIFIER] = ["value"]; // string, same as "name" when part of VAR 
        x[tkn.NUMBER] = ["value"]; // number (can differ from srcs)
        x[tkn.REGEXP] = ["value"]; // string
        x[tkn.STRING] = ["value"]; // string (can differ from srcs)
*/

        var c = [
            /*[stmt]*/
            tkn.SCRIPT, tkn.BLOCK,

            /*[expr]*/
            tkn.COMMA,
<commit_msg>LABEL node _label property is a traversable IDENTIFIER<commit_after>"use strict"; "use restrict";

var require = require || function(f) { load(f); };
require.paths && typeof __dirname !== "undefined" && require.paths.unshift(__dirname);
/* global Narcissus, tkn */ (typeof Narcissus === "undefined") && require("narcissus.js");
var Fmt = Fmt || require("fmt.js") || Fmt;
var Ref = Ref || require("ref.js") || Ref;
var Log = Log || require("log.js") || Log;
var Assert = Assert || require("assert.js") || Assert;
var Comments = Comments || require("comments.js") || Comments;
var log = (typeof console !== "undefined") && console.log || print;

var Shaper = (function() {
    Array.isArray = Array.isArray || function(o) {
        return Object.prototype.toString.call(o) === "[object Array]";
    };
    function error(node, msg) {
        log(Fmt("{0}:{1} error: {2}", node.tokenizer.filename, node.lineno, msg));
        (typeof quit === "undefined" ? process.exit : quit)(-1);
    }
    function deprecated(obj, params) {
        obj[params.was] = function() {
            throw new Error(Fmt.obj((params.now ?
                                     "{was} is deprecated since version {since}, use {now} instead" :
                                     "{was} is deprecated since version {since}"), params));
        };
    }

    var traverseData = (function() {
        var o = [];

        o[tkn.ARRAY_COMP] = [/*expr*/"expression", /*COMP_TAIL*/"tail"];
        o[tkn.CASE] = [/*expr*/"caseLabel", /*BLOCK*/"statements"];
        o[tkn.CATCH] = [/*IDENTIFIER*/"_name", /*expr*/"guard", /*BLOCK*/"block"];
        o[tkn.COMP_TAIL] = [/*[FOR_IN]*/"children", /*expr*/"guard"]; // has children but not last
        o[tkn.DEFAULT] = [/*BLOCK*/"statements"];
        o[tkn.DO] = [/*stmt*/"body", /*expr*/"condition"];
        o[tkn.FOR] = [/*expr*/"setup", /*expr*/"condition", /*expr*/"update", /*stmt*/"body"];
        o[tkn.FOR_IN] = [/*IDENTIFIER|VAR*/"_iterator", /*expr*/"object", /*stmt*/"body"];
        o[tkn.FUNCTION] = [/*IDENTIFIER*/"_name", /*[IDENTIFIER]*/"_params", /*SCRIPT*/"body"];
        o[tkn.GENERATOR] = [/*expr*/"expression", /*COMP_TAIL*/"tail"];
        o[tkn.GETTER] = [/*SCRIPT*/"body"];
        o[tkn.IF] = [/*expr*/"condition", /*stmt*/"thenPart", /*stmt*/"elsePart"];
        o[tkn.LABEL] = [/*IDENTIFIER*/"_label", /*stmt*/"statement"];
        o[tkn.LET_BLOCK] = [/*LET*/"variables", /*expr*/"expression", /*BLOCK*/"block"];
        o[tkn.RETURN] = [/*expr*/"value"];
        o[tkn.SEMICOLON] = [/*expr*/"expression"];
        o[tkn.SETTER] = [/*SCRIPT*/"body"];
        o[tkn.SWITCH] = [/*expr*/"discriminant", /*[CASE|DEFAULT]*/"cases"];
        o[tkn.THROW] = [/*expr*/"exception"];
        o[tkn.TRY] = [/*BLOCK*/"tryBlock", /*[CATCH]*/"catchClauses", /*BLOCK*/"finallyBlock"];
        o[tkn.WHILE] = [/*expr*/"condition", /*stmt*/"body"];
        o[tkn.WITH] = [/*expr*/"object", /*stmt*/"body"];
        o[tkn.YIELD] = [/*expr*/"value"];

/*
        // TODO these properties aren't nodes but may still be relevant
        x[tkn.ASSIGN] = ["assignOp"]; // number ("value" has string representation)
        x[tkn.INCREMENT] = ["postfix"]; // boolean ("value" is just "++")
        x[tkn.DECREMENT] = ["postfix"]; // boolean
        x[tkn.FUNCTION] = ["functionForm"]; // number
        x[tkn.FOR_IN] = ["isEach"]; // boolean
        x[tkn.SWITCH] = ["defaultIndex"]; // number
        x[tkn.IDENTIFIER] = ["value"]; // string, same as "name" when part of VAR 
        x[tkn.NUMBER] = ["value"]; // number (can differ from srcs)
        x[tkn.REGEXP] = ["value"]; // string
        x[tkn.STRING] = ["value"]; // string (can differ from srcs)
*/

        var c = [
            /*[stmt]*/
            tkn.SCRIPT, tkn.BLOCK,

            /*[expr]*/
            tkn.COMMA,

{'fraction_non_alphanumeric': 0.231282722513089, 'fraction_numerical': 0.0010471204188481676, 'mean_word_length': 2.6333808844507844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1819943', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2565, 'n_words': 514}