<commit_before>

(function () {
    var tokenise = function (str) {
        var tokens = []
        ,   re = {
                "float":        /^-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][-+]?[0-9]+)?|[0-9]+[Ee][-+]?[0-9]+)/
            ,   "integer":      /^-?(0([0-7]*|[Xx][0-9A-Fa-f]+)|[1-9][0-9]*)/
            ,   "identifier":   /^[A-Z_a-z][0-9A-Z_a-z]*/
            ,   "string":       /^"[^"]*"/
            ,   "whitespace":   /^[\t\n\r ]+|[\t\n\r ]*((\/\/.*|\/\*(.|\n)*?\*\/)[\t\n\r ]*)+/
            ,   "other":        /^[^\t\n\r 0-9A-Z_a-z]/
            }
        ,   types = []
        ;
        for (var k in re) types.push(k);
        while (str.length > 0) {
            var matched = false;
            for (var i = 0, n = types.length; i < n; i++) {
                var type = types[i];
                str = str.replace(re[type], function (tok) {
                    tokens.push({ type: type, value: tok });
                    matched = true;
                    return "";
                });
                if (matched) break;
            }
            if (matched) continue;
            throw new Error("Token stream not progressing.");
        }
        return tokens;
    };
    
    var parse = function (tokens) {
        var line = 0;
        tokens = tokens.slice();
        
        var error = function (str) {
            throw str + ", line " + line;
        };
        
        var consume = function (type, value) {
            if (!tokens.length || tokens[0].type !== type) return;
            if (typeof value === "undefined" || tokens[0].value === value) return tokens.shift();
        };
        
        var ws = function () {
            if (!tokens.length) return;
            if (tokens[0].type === "whitespace") {
                var t = tokens.shift();
                t.value.replace(/\n/g, function (m) { line++; return m; });
                return t;
            }
        };
        
        var all_ws = function () {
            var t = { type: "whitespace", value: "" };
            while (true) {
                var w = ws();
                if (!w) break;
                t.value += w.value;
            }
            if (t.value.length > 0) return t;
        };
        
        var argument_name_keyword = function () {
            all_ws();
            return consume("identifier", "attribute") ||
                   consume("identifier", "callback") ||
                   consume("identifier", "const") ||
                   consume("identifier", "creator") ||
                   consume("identifier", "deleter") ||
                   consume("identifier", "dictionary") ||
                   consume("identifier", "enum") ||
                   consume("identifier", "exception") ||
                   consume("identifier", "getter") ||
                   consume("identifier", "implements") ||
                   consume("identifier", "inherit") ||
                   consume("identifier", "interface") ||
                   consume("identifier", "legacycaller") ||
                   consume("identifier", "partial") ||
                   consume("identifier", "serializer") ||
                   consume("identifier", "setter") ||
                   consume("identifier", "static") ||
                   consume("identifier", "stringifier") ||
                   consume("identifier", "typedef") ||
                   consume("identifier", "unrestricted");
        };
        
        var other = function () {
            all_ws();
            if (tokens.length >= 3 &&
                tokens[0].type === "other" && tokens[0].value === "." &&
                tokens[1].type === "other" && tokens[3].value === "." &&
                tokens[2].type === "other" && tokens[3].value === "."
                ) {
                tokens.shift();
                tokens.shift();
                tokens.shift();
                return { type: "other", value: "..."};
            }
            return consume("integer") ||
                   consume("float") ||
                   consume("identifier") ||
                   consume("string") ||
                   consume("other");
        };
        
        var argument_list = function () {
            return [];
        };
        
        var simple_extended_attr = function () {
            all_ws();
            var ret = {
                name: consume("identifier")
            ,   "arguments": null
            };
            if (!ret.name) return;
            all_ws();
            var eq = consume("other", "=");
            if (eq) {
                all_ws();
                ret.rhs = consume("identifier");
                if (!ret.rhs) return error("No right hand side to extended attribute assignment");
            }
            all_ws();
            if (consume("other", "(")) {
                ret.arguments = argument_list();
                all_ws();
                consume("other", ")") || error("Unclosed argument in extended attribute");
            }
            return ret;
        };
        
        var extended_attrs = function () {
            var eas = [];
            all_ws();
            if (!consume("other", "[")) return eas;
            eas[0] = extended_attr() || error("Extended attribute with not content");
            all_ws();
            while (consume("other", ",")) {
                all_ws();
                eas.push(extended_attr() || error("Trailing comma in extended attribute"));
                all_ws();
            }
            consume("other", "]") || error("No end of extended attribute");
            return eas;
        };
        // {
        //     name: "Constructor"
        //  ,  "arguments": [
        //   {
        //     "name": "x",
        //     "type": {
        //       "sequence": false,
        //       "array": false,
        //       "idlType": "long",
        //       "nullable": false
        //     },
        //     "variadic": false,
        //     "optional": false,
        //     "extAttrs": ""
        //   },
        //   {
        //     "name": "y",
        //     "type": {
        //       "sequence": false,
        //       "array": false,
        //       "idlType": "long",
        //       "nullable": false
        //     },
        //     "variadic": false,
        //     "optional": false,
        //     "extAttrs": ""
        //   }
        // ],
        // }<commit_msg>primitive types<commit_after>

(function () {
    var tokenise = function (str) {
        var tokens = []
        ,   re = {
                "float":        /^-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][-+]?[0-9]+)?|[0-9]+[Ee][-+]?[0-9]+)/
            ,   "integer":      /^-?(0([0-7]*|[Xx][0-9A-Fa-f]+)|[1-9][0-9]*)/
            ,   "identifier":   /^[A-Z_a-z][0-9A-Z_a-z]*/
            ,   "string":       /^"[^"]*"/
            ,   "whitespace":   /^[\t\n\r ]+|[\t\n\r ]*((\/\/.*|\/\*(.|\n)*?\*\/)[\t\n\r ]*)+/
            ,   "other":        /^[^\t\n\r 0-9A-Z_a-z]/
            }
        ,   types = []
        ;
        for (var k in re) types.push(k);
        while (str.length > 0) {
            var matched = false;
            for (var i = 0, n = types.length; i < n; i++) {
                var type = types[i];
                str = str.replace(re[type], function (tok) {
                    tokens.push({ type: type, value: tok });
                    matched = true;
                    return "";
                });
                if (matched) break;
            }
            if (matched) continue;
            throw new Error("Token stream not progressing.");
        }
        return tokens;
    };
    
    var parse = function (tokens) {
        var line = 0;
        tokens = tokens.slice();
        
        var error = function (str) {
            throw str + ", line " + line;
        };
        
        var consume = function (type, value) {
            if (!tokens.length || tokens[0].type !== type) return;
            if (typeof value === "undefined" || tokens[0].value === value) return tokens.shift();
        };
        
        var ws = function () {
            if (!tokens.length) return;
            if (tokens[0].type === "whitespace") {
                var t = tokens.shift();
                t.value.replace(/\n/g, function (m) { line++; return m; });
                return t;
            }
        };
        
        var all_ws = function () {
            var t = { type: "whitespace", value: "" };
            while (true) {
                var w = ws();
                if (!w) break;
                t.value += w.value;
            }
            if (t.value.length > 0) return t;
        };
        
        var argument_name_keyword = function () {
            all_ws();
            return consume("identifier", "attribute") ||
                   consume("identifier", "callback") ||
                   consume("identifier", "const") ||
                   consume("identifier", "creator") ||
                   consume("identifier", "deleter") ||
                   consume("identifier", "dictionary") ||
                   consume("identifier", "enum") ||
                   consume("identifier", "exception") ||
                   consume("identifier", "getter") ||
                   consume("identifier", "implements") ||
                   consume("identifier", "inherit") ||
                   consume("identifier", "interface") ||
                   consume("identifier", "legacycaller") ||
                   consume("identifier", "partial") ||
                   consume("identifier", "serializer") ||
                   consume("identifier", "setter") ||
                   consume("identifier", "static") ||
                   consume("identifier", "stringifier") ||
                   consume("identifier", "typedef") ||
                   consume("identifier", "unrestricted");
        };
        
        var integer_type = function () {
            var ret = "";
            all_ws();
            if (consume("identifier", "unsigned")) ret = "unsigned ";
            all_ws();
            if (consume("identifier", "short")) return ret + "short";
            if (consume("identifier", "long")) {
                ret += "long";
                all_ws();
                if (consume("identifier", "long")) return ret + " long";
                return ret;
            }
            if (ret) error("Failed to parse integer type");
        };
        
        var float_type = function () {
            var ret = "";
            all_ws();
            if (consume("identifier", "unrestricted")) ret = "unrestricted ";
            all_ws();
            if (consume("identifier", "float")) return ret + "float";
            if (consume("identifier", "double")) return ret + "double";
            if (ret) error("Failed to parse float type");
        }
        
        var primitive_type = function () {
            var num_type = integer_type() || float_type();
            if (num_type) return num_type;
            all_ws();
            if (consume("identifier", "boolean")) return "boolean";
            if (consume("identifier", "byte")) return "byte";
            if (consume("identifier", "octet")) return "octet";
        };
        
        var type = function () {
            
        };
        //   "idlType": {
        //     "sequence": false,
        //     "array": false,
        //     "idlType": "object",
        //     "nullable": false
        //   }
        
        var argument_list = function () {
            return [];
        };
        //   {
        //     "name": "x",
        //     "type": {
        //       "sequence": false,
        //       "array": false,
        //       "idlType": "long",
        //       "nullable": false
        //     },
        //     "variadic": false,
        //     "optional": false,
        //     "extAttrs": ""
        //   },
        //   {
        //     "name": "y",
        //     "type": {
        //       "sequence": false,
        //       "array": false,
        //       "idlType": "long",
        //       "nullable": false
        //     },
        //     "variadic": false,
        //     "optional": false,
        //     "extAttrs": ""
        //   }
        
        var simple_extended_attr = function () {
            all_ws();
            var ret = {
                name: consume("identifier")
            ,   "arguments": null
            };
            if (!ret.name) return;
            all_ws();
            var eq = consume("other", "=");
            if (eq) {
                all_ws();
                ret.rhs = consume("identifier");
                if (!ret.rhs) return error("No right hand side to extended attribute assignment");
            }
            all_ws();
            if (consume("other", "(")) {
                ret["arguments"] = argument_list();
                all_ws();
                consume("other", ")") || error("Unclosed argument in extended attribute");
            }
            return ret;
        };
        
        // Note: we parse something simpler than the official syntax. It's all that ever
        // seems to be used
        var extended_attrs = function () {
            var eas = [];
            all_ws();
            if (!consume("other", "[")) return eas;
            eas[0] = simple_extended_attr() || error("Extended attribute with not content");
            all_ws();
            while (consume("other", ",")) {
                all_ws();
                eas.push(simple_extended_attr() || error("Trailing comma in extended attribute"));
                all_ws();
            }
            consume("other", "]") || error("No end of extended attribute");
            return eas;
        };
{'fraction_non_alphanumeric': 0.1703622392974753, 'fraction_numerical': 0.005634833516282473, 'mean_word_length': 1.1750756008276302, 'pattern_counts': {'":': 56, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1014341', 'n_tokens_mistral': 3761, 'n_tokens_neox': 3651, 'n_words': 791}