<commit_before>JSONEX = {
    allow_async: false,
    functions: {},
    undefined_function: function(name) {
        return new JSONEX.FunctionCall(name, Array.prototype.slice.call(arguments, 1));
    }
};

JSONEX.configure = function(options) {
    if ("allow_async" in options)
        this.allow_async = options.allow_async;
    if ("undefined_function" in options)
        this.undefined_function = options.undefined_function;
    if ("inherit_functions" in options && !options.inherit_functions)
        this.functions = options.functions || {};
    else if ("functions" in options)
        for (var key in options.functions)
            if (options.hasOwnProperty(key))
                this.functions[key] = options.functions[key];
};

JSONEX.parser = function(options) {
    if (this.hasOwnProperty("functions"))
        throw "JSONEX.parser cannot be called as function (without new keyword)";
    this.functions = Object.create(this.functions);
    this.parser = function() { JSONEX.parser.apply(this, arguments); };
    this.parser.prototype = this;

    options && this.configure(options);
};
JSONEX.parser.prototype = JSONEX;

JSONEX.parse = function(str, options, ctx) {
    ctx = ctx || {};
    options = options ? new this.parser(options) : this;

    return JSON.parse(str, function (key, value) {
        if (key === "?")
            if (typeof value === "string") // { "?": "f", named_args }
                return [value];
            else if (value instanceof Array) { // { "?": ["f", args], named_args }
                return value;
            } else {
                console.warn && console.warn("Malformed JSONEX: '?' property must be string or array");
                return ["?", value];
            }

        if (typeof value === "object" && value && "?" in value) {
            var args = value["?"];

            if (args[0] == "?") {
                args.length == 2 || console.warn && console.warn("Malformed JSONEX: masked '?' property with wrong arguments number ");
                value["?"] = args[1];
            } else {
                if (Object.keys(value).length > 1) { // push named_args as last argument
                    delete value["?"];
                    args.push(value);
                }

                return ApplyFunction(args);
            }
        }

        return options.allow_async ? WrapAsync(value) : value;
    });

    function ApplyFunction(args) {
        var func;
        if (options.functions.hasOwnProperty(args[0]))       {
            func = options.functions[args.shift()];

        } else func = options.undefined_function;

        if (options.allow_async && !func.allow_input_promises) args = WrapAsync(args);

        if (typeof args.then === "function")
            return args.then(function(value) { return func.apply(ctx, value) });
        else
            return func.apply(ctx, args);
    }

    function WrapAsync(value) {
        if (typeof value != "object" || value === null) return value;

        var result = value instanceof Array ? [] : {};
        var last_promise = null;

        for (var key in value)
            if (value.hasOwnProperty(key))
            {
                if (value[key] && typeof value[key].then === "function")
                    last_promise = value[key].then(function(last_promise, key) {
                        return function (subresult) {
                            result[key] = subresult;
                            return last_promise;
                        };
                    }(last_promise, key));
                else
                    result[key] = value[key];
            }

        if (last_promise)
            return last_promise.then(function() { return result; });
        else
            return value;
    }
};

JSONEX.stringify = function(value, space) {
    return JSON.stringify(value, function(key, value) {
        if (key === "?" && !(value instanceof JSONEX.ArgumentList))
            return ["?", value];
        while (value && typeof value.toJSONEX === "function")
            value = value.toJSONEX();
        return value;
    }, space);
};

JSONEX.FunctionCall = function FunctionCall(name, args) {
    var result = "toJSONEX" in this ? this : object.create(JSONEX.FunctionCall.prototype);
    result.function = name;
    result.arguments = args;
    return result;
};

JSONEX.FunctionCall.prototype = {
    toJSONEX: function() {
        return { "?": new JSONEX.ArgumentList([this.function].concat(this.arguments)) };
    }
};

JSONEX.ArgumentList = function ArgumentList(args) {
    var result = "toJSONEX" in this ? this : object.create(JSONEX.ArgumentList.prototype);
    this.args = args;
};

JSONEX.ArgumentList.prototype = {
    toJSONEX: function() {
        return this.args;
    }
};<commit_msg>fixed: JSONEX.ArgumentList cannot be called as function<commit_after>JSONEX = {
    allow_async: false,
    functions: {},
    undefined_function: function(name) {
        return new JSONEX.FunctionCall(name, Array.prototype.slice.call(arguments, 1));
    }
};

JSONEX.configure = function(options) {
    if ("allow_async" in options)
        this.allow_async = options.allow_async;
    if ("undefined_function" in options)
        this.undefined_function = options.undefined_function;
    if ("inherit_functions" in options && !options.inherit_functions)
        this.functions = options.functions || {};
    else if ("functions" in options)
        for (var key in options.functions)
            if (options.hasOwnProperty(key))
                this.functions[key] = options.functions[key];
};

JSONEX.parser = function(options) {
    if (this.hasOwnProperty("functions"))
        throw "JSONEX.parser cannot be called as function (without new keyword)";
    this.functions = Object.create(this.functions);
    this.parser = function() { JSONEX.parser.apply(this, arguments); };
    this.parser.prototype = this;

    options && this.configure(options);
};
JSONEX.parser.prototype = JSONEX;

JSONEX.parse = function(str, options, ctx) {
    ctx = ctx || {};
    options = options ? new this.parser(options) : this;

    return JSON.parse(str, function (key, value) {
        if (key === "?")
            if (typeof value === "string") // { "?": "f", named_args }
                return [value];
            else if (value instanceof Array) { // { "?": ["f", args], named_args }
                return value;
            } else {
                console.warn && console.warn("Malformed JSONEX: '?' property must be string or array");
                return ["?", value];
            }

        if (typeof value === "object" && value && "?" in value) {
            var args = value["?"];

            if (args[0] == "?") {
                args.length == 2 || console.warn && console.warn("Malformed JSONEX: masked '?' property with wrong arguments number ");
                value["?"] = args[1];
            } else {
                if (Object.keys(value).length > 1) { // push named_args as last argument
                    delete value["?"];
                    args.push(value);
                }

                return ApplyFunction(args);
            }
        }

        return options.allow_async ? WrapAsync(value) : value;
    });

    function ApplyFunction(args) {
        var func;
        if (options.functions.hasOwnProperty(args[0]))       {
            func = options.functions[args.shift()];

        } else func = options.undefined_function;

        if (options.allow_async && !func.allow_input_promises) args = WrapAsync(args);

        if (typeof args.then === "function")
            return args.then(function(value) { return func.apply(ctx, value) });
        else
            return func.apply(ctx, args);
    }

    function WrapAsync(value) {
        if (typeof value != "object" || value === null) return value;

        var result = value instanceof Array ? [] : {};
        var last_promise = null;

        for (var key in value)
            if (value.hasOwnProperty(key))
            {
                if (value[key] && typeof value[key].then === "function")
                    last_promise = value[key].then(function(last_promise, key) {
                        return function (subresult) {
                            result[key] = subresult;
                            return last_promise;
                        };
                    }(last_promise, key));
                else
                    result[key] = value[key];
            }

        if (last_promise)
            return last_promise.then(function() { return result; });
        else
            return value;
    }
};

JSONEX.stringify = function(value, space) {
    return JSON.stringify(value, function(key, value) {
        if (key === "?" && !(value instanceof JSONEX.ArgumentList))
            return ["?", value];
        while (value && typeof value.toJSONEX === "function")
            value = value.toJSONEX();
        return value;
    }, space);
};

JSONEX.FunctionCall = function FunctionCall(name, args) {
    var result = "toJSONEX" in this ? this : object.create(JSONEX.FunctionCall.prototype);
    result.function = name;
    result.arguments = args;
    return result;
};

JSONEX.FunctionCall.prototype = {
    toJSONEX: function() {
        return { "?": new JSONEX.ArgumentList([this.function].concat(this.arguments)) };
    }
};

JSONEX.ArgumentList = function ArgumentList(args) {
    var result = "toJSONEX" in this ? this : object.create(JSONEX.ArgumentList.prototype);
    result.args = args;
    return result;
};

JSONEX.ArgumentList.prototype = {
    toJSONEX: function() {
        return this.args;
    }
};
{'fraction_non_alphanumeric': 0.13231472819873194, 'fraction_numerical': 0.0012472715933894605, 'mean_word_length': 2.08793324775353, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5848944', 'n_tokens_mistral': 2657, 'n_tokens_neox': 2586, 'n_words': 631}