<commit_before>
// Base AST class.

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST = function() {};

Yate.AST._asts = {};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.options = {};

Yate.AST.prototype.$rid = 0;
Yate.AST.prototype.$cid = 0;

Yate.AST.prototype._init = function() {};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.error = function(s) {
    throw 'ERROR: ' + s + '\n' + Yate.Parser._where(this.where);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.make = function(id) {
    var ast = new (Yate.AST.$(id))();
    ast.id = id; // Если делать это в прототипе, то не видно в console.log.
    ast._init.apply(ast, Array.prototype.slice.call(arguments, 1));
    return ast;
};

Yate.AST.prototype.make = function() {
    var ast = Yate.AST.make.apply(null, arguments);
    ast.parent = this;
    ast.setLocals();
    return ast;
};


Yate.AST.$ = function(id) {
    var ast = this._asts[id];

    if (!ast) {

        ast = function() {};

        var info = Yate.AST[id] || {};
        var options = info.options = info.options || {};

        var base = (options.base) ? this.$(options.base) : Yate.AST;
        var mixin = [];
        if (options.mixin) {
            options.mixin = Yate.Common.makeArray(options.mixin);
            mixin = mixin.concat(Yate.Common.map(options.mixin, function(id) { return Yate.AST[id] || {}; }));
        }
        mixin.push(info);
        Yate.Common.inherits(ast, base, mixin);

        this._asts[id] = ast;
    }

    return ast;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.yate = function(id, data) {
    return this._yate(id, data);
};

Yate.AST.prototype._yate = function(id, data) {
    return this._fill('yate', id, data);
};

Yate.AST.prototype.js = function(id, data) {
    return this._js(id, data);
};

Yate.AST.prototype._js = function(id, data) {
    return this._fill('js', id, data);
};

Yate.AST.prototype._fill = function(type, id, data) {
    if (!id) {
        id = this.id;
        data = this;
    } else if (!(id && data)) {
        if (typeof id == 'string') {
            data = this;
        } else {
            data = id;
            id = this.id;
        }
    }

    return Yate.CodeTemplates.fill(type, id, data, type);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.children = function() {
    var children = [];
    var order = this.options.order;
    if (order) {
        for (var i = 0, l = order.length; i < l; i++) {
            children.push(this['$' + order[i]]);
        }
    } else {
        for (var key in this) {
            if (this.hasOwnProperty(key) && /^\$/.test(key)) {
                children.push(this[key]);
            }
        }
    }
    return children;
};

Yate.AST.prototype._apply = function(callback) {
    var args = Array.prototype.slice.call(arguments, 1);

    var children = this.children();
    for (var i = 0, l = children.length; i < l; i++) {
        step(children[i]);
    }

    function step(child) {
        if (child instanceof Array) {
            for (var j = 0, m = child.length; j < m; j++) {
                step(child[j]);
            }
        } else if (child instanceof Yate.AST) {
            callback.apply(child, args);
        }
    }
};

Yate.AST.prototype.trigger = function(method) {
    var args = Array.prototype.slice.call(arguments, 1);

    run.apply(this, args);

    function run() {
        var callback;
        if ((typeof method == 'function')) {
            callback = method;
        } else if (typeof method == 'string' && typeof this[method] == 'function') {
            callback = this[method];
        }
        var args = Array.prototype.slice.call(arguments);
        if (callback) {
            callback.apply(this, args);
        }
        this._apply.apply(this, [ run ].concat(args));
    };
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.setParents = function() {
    function set(parent) {
        this.parent = parent;
        this._apply(set, this);
    }

    set.call(this, null);
};

// ----------------------------------------------------------------------------------------------------------------- //
// Type methods
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.type = function() {
    var type = this._type;
    if (type === undefined) {
        type = this._type = this._getType();
    }
    return type;
};

Yate.AST.prototype._getType = function() {
    this.error('Yate.AST.' + this.id + '.prototype._getType is not defined');
};

Yate.AST.prototype.cast = function(to) {
    var from = this.type();
    this._asType = to = to || from;

    if (!Yate.Types.convertable(from, to)) {
        this.error('Cannot convert type from ' + from + ' to ' + to + ' ' + this.id);
    }

    return this.oncast(from, to) || this;
};

Yate.AST.prototype.oncast = function(from, to) {
    var items = this.$items;
    if (items) {
        for (var i = 0, l = items.length; i < l; i++) {
            items[i] = items[i].cast(to);
        }
    }
};

Yate.AST.prototype.is = function(type) {
    if (type instanceof Array) {
        for (var i = 0, l = type.length; i < l; i++) {
            if (this instanceof Yate.AST.$(type[i])) {
                return true;
            }
        }
    } else {
        return this instanceof Yate.AST.$(type);
    }
};

// ----------------------------------------------------------------------------------------------------------------- //
// Locals: state, context, scope
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local = function() {
};

Yate.AST.Local.prototype.child = function() {
    var local = new this.constructor();
    local.parent = this;
    return local;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.State = function() {
    this.jpaths = [];
    this.jkeys = {};

    this.predicates = [];
    this.pkeys = {};

    // Глобальные id-шники:
    this.jid = 0; // jpath'ы.
    this.pid = 0; // Предикаты.
    this.tid = 0; // Шаблоны.
    this.vid = 0; // Переменные.
    this.fid = 0; // Функции.
    this.kid = 0; // Ключи.
};

Yate.Common.inherits(Yate.AST.Local.State, Yate.AST.Local);

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.Context = function() {
};

Yate.Common.inherits(Yate.AST.Local.Context, Yate.AST.Local);

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.Scope = function() {
    this.vars = {};
    this.functions = {};
    this.id = Yate.AST.Local.Scope._id++;
};

Yate.AST.Local.Scope._id = 0;

Yate.Common.inherits(Yate.AST.Local.Scope, Yate.AST.Local);

Yate.AST.Local.Scope.prototype.findVar = function(name) {
    var scope = this;
    while (scope) {
        var value = scope.vars[name];
        if (value) {
            return value;
        }
        scope = scope.parent;
    }
};

Yate.AST.Local.Scope.prototype.findFunction = function(name) {
    var scope = this;
    while (scope) {
        var value = scope.functions[name];
        if (value) {
            return value;
        }
        scope = scope.parent;
    }
    return Yate.AST.internalFunctions[name]; // Если ничего не нашли в scope'ах, смотрим на список встроенных функций.
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.locals = {
    state: Yate.AST.Local.State,
    context: Yate.AST.Local.Context,
    scope: Yate.AST.Local.Scope
};

// ----------------------------------------------------------------------------------------------------------------- //
// Walk methods
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.setLocals = function() {
    var options = this.options.locals || {};
    var locals = Yate.AST.locals;
    var parent = this.parent;

    for (var local in locals) {
        var parentLocal = (parent) ? parent[local] : null;
        if (options[local]) {
            this[local] = (parentLocal) ? parentLocal.child() : new locals[local]();
        } else {
            this[local] = parentLocal;
        }
    }

    this.$sid = this.scope.id;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.log = function() {
    console.log(this.id, this.parent && this.parent.id, this.scope && this.scope.id);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.set = function(key, value) {
    this.trigger(function() {
        this[key] = value;
    });
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.isOpen = function() {
    if (this.type() == Yate.Types.ATTR || this.type() == Yate.Types.XML) {
        return undefined;
    }
    return false;
};

// ----------------------------------------------------------------------------------------------------------------- //
// Items
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.items = {

    _init: function(items) {
        this.$items = Yate.Common.makeArray(items || []);
    },

    _getType: function() {
        var items = this.$items;

        var l = items.length;
        if (!l) { return Yate.Types.SCALAR; }

        var type = items[0].type();

        for (var i = 1; i < l; i++) {
            type = Yate.Types.joinType(type, items[i].type());
            if (type == Yate.Types.NONE) { return Yate.Types.NONE; }
        }

        return type;
    },

    add: function(item) {
        this.$items.push(item);
    },

    items: function() {
        return this.$items;
    },

    last: function() {
        var items = this.$items;
        return items[items.length - 1];
    },

    empty: function() {
        return (this.$items.length == 0);
    },

    iterate: function(callback) {
        var items = this.$items;
        for (var i = 0, l = items.length; i < l; i++) {
            callback(items[i], i);
        }
    },

    grep: function(callback) {
        var items = this.$items;
        var r = [];
        for (var i = 0, l = items.length; i < l; i++) {
            var item = items[i];
            if (callback(item, i)) {
                r.push(item);
            }
        }
        return r;
    },

    map: function(callback) {
        return Yate.Common.map(this.$items, callback);
    },

    yate: function() {
        var options = this.options.yate || {};
        return this.map('yate').join(options.separator || '');
    },

    js: function() {
        var options = this.options.js || {};
        return this.map('js').join(options.separator || '');
    },

    toResult: function(result) {
        this.iterate(function(item) {
            item.toResult(result);
        });
    }

};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.varType = {
    USER: 'user',
    ARGUMENT: 'argument'
};

Yate.AST.functionType = {
    USER: 'user',
    INTERNAL: 'internal',
    EXTERNAL: 'external',
    KEY: 'key'
};

<commit_msg>Выводим stacktrace в error()<commit_after>
// Base AST class.

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST = function() {};

Yate.AST._asts = {};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.options = {};

Yate.AST.prototype.$rid = 0;
Yate.AST.prototype.$cid = 0;

Yate.AST.prototype._init = function() {};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.error = function(s) {
    require('util').puts(new Error().stack);
    throw 'ERROR: ' + s + '\n' + Yate.Parser._where(this.where);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.make = function(id) {
    var ast = new (Yate.AST.$(id))();
    ast.id = id; // Если делать это в прототипе, то не видно в console.log.
    ast._init.apply(ast, Array.prototype.slice.call(arguments, 1));
    return ast;
};

Yate.AST.prototype.make = function() {
    var ast = Yate.AST.make.apply(null, arguments);
    ast.parent = this;
    ast.setLocals();
    return ast;
};


Yate.AST.$ = function(id) {
    var ast = this._asts[id];

    if (!ast) {

        ast = function() {};

        var info = Yate.AST[id] || {};
        var options = info.options = info.options || {};

        var base = (options.base) ? this.$(options.base) : Yate.AST;
        var mixin = [];
        if (options.mixin) {
            options.mixin = Yate.Common.makeArray(options.mixin);
            mixin = mixin.concat(Yate.Common.map(options.mixin, function(id) { return Yate.AST[id] || {}; }));
        }
        mixin.push(info);
        Yate.Common.inherits(ast, base, mixin);

        this._asts[id] = ast;
    }

    return ast;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.yate = function(id, data) {
    return this._yate(id, data);
};

Yate.AST.prototype._yate = function(id, data) {
    return this._fill('yate', id, data);
};

Yate.AST.prototype.js = function(id, data) {
    return this._js(id, data);
};

Yate.AST.prototype._js = function(id, data) {
    return this._fill('js', id, data);
};

Yate.AST.prototype._fill = function(type, id, data) {
    if (!id) {
        id = this.id;
        data = this;
    } else if (!(id && data)) {
        if (typeof id == 'string') {
            data = this;
        } else {
            data = id;
            id = this.id;
        }
    }

    return Yate.CodeTemplates.fill(type, id, data, type);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.children = function() {
    var children = [];
    var order = this.options.order;
    if (order) {
        for (var i = 0, l = order.length; i < l; i++) {
            children.push(this['$' + order[i]]);
        }
    } else {
        for (var key in this) {
            if (this.hasOwnProperty(key) && /^\$/.test(key)) {
                children.push(this[key]);
            }
        }
    }
    return children;
};

Yate.AST.prototype._apply = function(callback) {
    var args = Array.prototype.slice.call(arguments, 1);

    var children = this.children();
    for (var i = 0, l = children.length; i < l; i++) {
        step(children[i]);
    }

    function step(child) {
        if (child instanceof Array) {
            for (var j = 0, m = child.length; j < m; j++) {
                step(child[j]);
            }
        } else if (child instanceof Yate.AST) {
            callback.apply(child, args);
        }
    }
};

Yate.AST.prototype.trigger = function(method) {
    var args = Array.prototype.slice.call(arguments, 1);

    run.apply(this, args);

    function run() {
        var callback;
        if ((typeof method == 'function')) {
            callback = method;
        } else if (typeof method == 'string' && typeof this[method] == 'function') {
            callback = this[method];
        }
        var args = Array.prototype.slice.call(arguments);
        if (callback) {
            callback.apply(this, args);
        }
        this._apply.apply(this, [ run ].concat(args));
    };
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.setParents = function() {
    function set(parent) {
        this.parent = parent;
        this._apply(set, this);
    }

    set.call(this, null);
};

// ----------------------------------------------------------------------------------------------------------------- //
// Type methods
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.type = function() {
    var type = this._type;
    if (type === undefined) {
        type = this._type = this._getType();
    }
    return type;
};

Yate.AST.prototype._getType = function() {
    this.error('Yate.AST.' + this.id + '.prototype._getType is not defined');
};

Yate.AST.prototype.cast = function(to) {
    var from = this.type();
    this._asType = to = to || from;

    if (!Yate.Types.convertable(from, to)) {
        this.error('Cannot convert type from ' + from + ' to ' + to + ' ' + this.id);
    }

    return this.oncast(from, to) || this;
};

Yate.AST.prototype.oncast = function(from, to) {
    var items = this.$items;
    if (items) {
        for (var i = 0, l = items.length; i < l; i++) {
            items[i] = items[i].cast(to);
        }
    }
};

Yate.AST.prototype.is = function(type) {
    if (type instanceof Array) {
        for (var i = 0, l = type.length; i < l; i++) {
            if (this instanceof Yate.AST.$(type[i])) {
                return true;
            }
        }
    } else {
        return this instanceof Yate.AST.$(type);
    }
};

// ----------------------------------------------------------------------------------------------------------------- //
// Locals: state, context, scope
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local = function() {
};

Yate.AST.Local.prototype.child = function() {
    var local = new this.constructor();
    local.parent = this;
    return local;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.State = function() {
    this.jpaths = [];
    this.jkeys = {};

    this.predicates = [];
    this.pkeys = {};

    // Глобальные id-шники:
    this.jid = 0; // jpath'ы.
    this.pid = 0; // Предикаты.
    this.tid = 0; // Шаблоны.
    this.vid = 0; // Переменные.
    this.fid = 0; // Функции.
    this.kid = 0; // Ключи.
};

Yate.Common.inherits(Yate.AST.Local.State, Yate.AST.Local);

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.Context = function() {
};

Yate.Common.inherits(Yate.AST.Local.Context, Yate.AST.Local);

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.Local.Scope = function() {
    this.vars = {};
    this.functions = {};
    this.id = Yate.AST.Local.Scope._id++;
};

Yate.AST.Local.Scope._id = 0;

Yate.Common.inherits(Yate.AST.Local.Scope, Yate.AST.Local);

Yate.AST.Local.Scope.prototype.findVar = function(name) {
    var scope = this;
    while (scope) {
        var value = scope.vars[name];
        if (value) {
            return value;
        }
        scope = scope.parent;
    }
};

Yate.AST.Local.Scope.prototype.findFunction = function(name) {
    var scope = this;
    while (scope) {
        var value = scope.functions[name];
        if (value) {
            return value;
        }
        scope = scope.parent;
    }
    return Yate.AST.internalFunctions[name]; // Если ничего не нашли в scope'ах, смотрим на список встроенных функций.
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.locals = {
    state: Yate.AST.Local.State,
    context: Yate.AST.Local.Context,
    scope: Yate.AST.Local.Scope
};

// ----------------------------------------------------------------------------------------------------------------- //
// Walk methods
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.setLocals = function() {
    var options = this.options.locals || {};
    var locals = Yate.AST.locals;
    var parent = this.parent;

    for (var local in locals) {
        var parentLocal = (parent) ? parent[local] : null;
        if (options[local]) {
            this[local] = (parentLocal) ? parentLocal.child() : new locals[local]();
        } else {
            this[local] = parentLocal;
        }
    }

    this.$sid = this.scope.id;
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.log = function() {
    console.log(this.id, this.parent && this.parent.id, this.scope && this.scope.id);
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.set = function(key, value) {
    this.trigger(function() {
        this[key] = value;
    });
};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.prototype.isOpen = function() {
    if (this.type() == Yate.Types.ATTR || this.type() == Yate.Types.XML) {
        return undefined;
    }
    return false;
};

// ----------------------------------------------------------------------------------------------------------------- //
// Items
// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.items = {

    _init: function(items) {
        this.$items = Yate.Common.makeArray(items || []);
    },

    _getType: function() {
        var items = this.$items;

        var l = items.length;
        if (!l) { return Yate.Types.SCALAR; }

        var type = items[0].type();

        for (var i = 1; i < l; i++) {
            type = Yate.Types.joinType(type, items[i].type());
            if (type == Yate.Types.NONE) { return Yate.Types.NONE; }
        }

        return type;
    },

    add: function(item) {
        this.$items.push(item);
    },

    items: function() {
        return this.$items;
    },

    last: function() {
        var items = this.$items;
        return items[items.length - 1];
    },

    empty: function() {
        return (this.$items.length == 0);
    },

    iterate: function(callback) {
        var items = this.$items;
        for (var i = 0, l = items.length; i < l; i++) {
            callback(items[i], i);
        }
    },

    grep: function(callback) {
        var items = this.$items;
        var r = [];
        for (var i = 0, l = items.length; i < l; i++) {
            var item = items[i];
            if (callback(item, i)) {
                r.push(item);
            }
        }
        return r;
    },

    map: function(callback) {
        return Yate.Common.map(this.$items, callback);
    },

    yate: function() {
        var options = this.options.yate || {};
        return this.map('yate').join(options.separator || '');
    },

    js: function() {
        var options = this.options.js || {};
        return this.map('js').join(options.separator || '');
    },

    toResult: function(result) {
        this.iterate(function(item) {
            item.toResult(result);
        });
    }

};

// ----------------------------------------------------------------------------------------------------------------- //

Yate.AST.varType = {
    USER: 'user',
    ARGUMENT: 'argument'
};

Yate.AST.functionType = {
    USER: 'user',
    INTERNAL: 'internal',
    EXTERNAL: 'external',
    KEY: 'key'
};


{'fraction_non_alphanumeric': 0.3497135842880524, 'fraction_numerical': 0.0018821603927986906, 'mean_word_length': 1.0783163265306122, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5339163', 'n_tokens_mistral': 6715, 'n_tokens_neox': 6608, 'n_words': 1345}