<commit_before>var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var path = require('path');
var _ = require('lodash');
var mkdirp = Promise.promisify(require('mkdirp'));

var config = {
    'dryrun': false,
    'encoding': 'binary',
    'files': {
        'scaffile': '.scaf',
        'scafout': '.scafout',
        'scafctx': '.scafctx',
    },
    'prefix': '-- ',
    'special': '_',
    'ignore': [],
};

var keys = {
    'key': config.special + 'k',
    'value': config.special + 'v',
    'self': config.special + 'self',
    'root': config.special + 'root',
    'parent': config.special + 'parent',
};

function ConflictingError(target, files) {
    this.message = "Multiple files writing to the same place: '" + target + "' => [" + files.join(', ') + "]";
    this.target = target;
    this.files = files;
    this.name = "ConflictingError";
    Error.captureStackTrace(this, ConflictingError);
}
ConflictingError.prototype = Object.create(Error.prototype);
ConflictingError.prototype.constructor = ConflictingError;

function NotFoundError(pa) {
    this.message = "No such file or directory: " + pa;
    this.pa = pa;
    this.name = "NotFoundError";
    Error.captureStackTrace(this, NotFoundError);
}
NotFoundError.prototype = Object.create(Error.prototype);
NotFoundError.prototype.constructor = NotFoundError;

var files = {};

function build_recursive(obj, reader, writer, pa, parentname) {
    if (parentname === undefined) {
        parentname = '';
    }
    return Promise.resolve(reader.isdir(pa))
        .then(isdir => {
            if (isdir) {
                if (utils_ignored(pa, 'dir')) {
                    return;
                }
                return build_dir(obj, reader, writer, pa, parentname);
            } else {
                if (utils_ignored(pa, 'file')) {
                    return;
                }
                return build_file(obj, reader, writer, pa, parentname);
            }
        });
}

function utils_create_kv(k, v) {
    var tmp = {};
    tmp[keys.key] = k;
    tmp[keys.value] = v;
    return tmp;
}

function utils_parse_scope(obj, scopestr) {
    /* jshint -W083 */
    var scope = scopestr.replace('[]', '.[]').replace('{}', '.{}').replace(/^\./, '').split('.');
    var res = [obj];
    var old;
    for (var ch of scope) {
        old = res;
        res = [];
        for (var e of old) {
            var subres = [];
            if (ch == '[]') {
                if (_.isArray(e)) {
                    subres = subres.concat(e);
                } else if (_.isObject(e)) {
                    subres = subres.concat(_.values(_.omit(e, _.values(keys))));
                }
            } else if (ch == '{}') {
                if (_.isArray(e[keys.self])) {
                    _.transform(e[keys.self], (res, v, k) => {
                        res.push(utils_create_kv(k, v));
                    }, subres);
                } else if (_.isObject(e[keys.self])) {
                    _.transform(_.omit(e[keys.self], _.values(keys)), (result, v, k) => {
                        result.push(utils_create_kv(k, v));
                    }, subres);
                }
            } else {
                subres.push(e[ch]);
            }
            for (var e2 of subres) {
                res.push(utils_normailze_elem(e2, e));
            }
        }
    }
    return res;
}

function utils_normailze_elem(elem, parent) {
    parent = parent || elem[keys.parent];
    var root = parent?parent[keys.root] || parent:elem;
    if (!_.isObject(elem)) {
        var tmp = {};
        tmp[keys.self] = elem;
        elem = tmp;
    } else {
        elem[keys.self] = elem;
    }
    elem[keys.parent] = parent;
    elem[keys.root] = root;
    return elem;
}


function utils_parse_recursive(optss, res, include) {
    if (optss.length === 0)
        return res;
    var more = [];
    if (res === undefined) {
        res = [];
    }
    return Promise.resolve(optss)
        .map(opts => {
            return utils_parse_once(opts, include)
                .map(parsed => {
                    if (parsed.args.length === 0) {
                        res.push(parsed);
                    } else {
                        more.push(parsed);
                    }

		});
	})
        .then(_ => utils_parse_recursive(more, res, include));
}

function utils_parse_once(opts, include) {
    var res = [];
    if (opts.args.length === 0)
        return [opts];

    var el = opts.args[0];
    opts.args = opts.args.slice(1);
    var t;
    switch (el.type) {
        case 'scope':
            var ctxs = utils_parse_scope(opts.ctx, el.arg);
            for (var ctx of ctxs) {
                res.push({
                    'ctx': ctx,
                    'name': opts.name.slice(),
                    'args': opts.args.slice(),
                    'orig': opts.orig.slice(),
                });
            }
            break;
        case 'cond':
	    res = Promise.resolve(utils_apply_template(el.arg, opts.ctx, {'opts': opts}))
                .then(cond => {
                    if (cond)
                        return [opts];
	            else
		        return [];
		});
            break;
        case 'includable':
            if (include)<commit_msg>cond is evaluated directly<commit_after>var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var path = require('path');
var _ = require('lodash');
var mkdirp = Promise.promisify(require('mkdirp'));

var config = {
    'dryrun': false,
    'encoding': 'binary',
    'files': {
        'scaffile': '.scaf',
        'scafout': '.scafout',
        'scafctx': '.scafctx',
    },
    'prefix': '-- ',
    'special': '_',
    'ignore': [],
};

var keys = {
    'key': config.special + 'k',
    'value': config.special + 'v',
    'self': config.special + 'self',
    'root': config.special + 'root',
    'parent': config.special + 'parent',
};

function ConflictingError(target, files) {
    this.message = "Multiple files writing to the same place: '" + target + "' => [" + files.join(', ') + "]";
    this.target = target;
    this.files = files;
    this.name = "ConflictingError";
    Error.captureStackTrace(this, ConflictingError);
}
ConflictingError.prototype = Object.create(Error.prototype);
ConflictingError.prototype.constructor = ConflictingError;

function NotFoundError(pa) {
    this.message = "No such file or directory: " + pa;
    this.pa = pa;
    this.name = "NotFoundError";
    Error.captureStackTrace(this, NotFoundError);
}
NotFoundError.prototype = Object.create(Error.prototype);
NotFoundError.prototype.constructor = NotFoundError;

var files = {};

function build_recursive(obj, reader, writer, pa, parentname) {
    if (parentname === undefined) {
        parentname = '';
    }
    return Promise.resolve(reader.isdir(pa))
        .then(isdir => {
            if (isdir) {
                if (utils_ignored(pa, 'dir')) {
                    return;
                }
                return build_dir(obj, reader, writer, pa, parentname);
            } else {
                if (utils_ignored(pa, 'file')) {
                    return;
                }
                return build_file(obj, reader, writer, pa, parentname);
            }
        });
}

function utils_create_kv(k, v) {
    var tmp = {};
    tmp[keys.key] = k;
    tmp[keys.value] = v;
    return tmp;
}

function utils_parse_scope(obj, scopestr) {
    /* jshint -W083 */
    var scope = scopestr.replace('[]', '.[]').replace('{}', '.{}').replace(/^\./, '').split('.');
    var res = [obj];
    var old;
    for (var ch of scope) {
        old = res;
        res = [];
        for (var e of old) {
            var subres = [];
            if (ch == '[]') {
                if (_.isArray(e)) {
                    subres = subres.concat(e);
                } else if (_.isObject(e)) {
                    subres = subres.concat(_.values(_.omit(e, _.values(keys))));
                }
            } else if (ch == '{}') {
                if (_.isArray(e[keys.self])) {
                    _.transform(e[keys.self], (res, v, k) => {
                        res.push(utils_create_kv(k, v));
                    }, subres);
                } else if (_.isObject(e[keys.self])) {
                    _.transform(_.omit(e[keys.self], _.values(keys)), (result, v, k) => {
                        result.push(utils_create_kv(k, v));
                    }, subres);
                }
            } else {
                subres.push(e[ch]);
            }
            for (var e2 of subres) {
                res.push(utils_normailze_elem(e2, e));
            }
        }
    }
    return res;
}

function utils_normailze_elem(elem, parent) {
    parent = parent || elem[keys.parent];
    var root = parent?parent[keys.root] || parent:elem;
    if (!_.isObject(elem)) {
        var tmp = {};
        tmp[keys.self] = elem;
        elem = tmp;
    } else {
        elem[keys.self] = elem;
    }
    elem[keys.parent] = parent;
    elem[keys.root] = root;
    return elem;
}


function utils_parse_recursive(optss, res, include) {
    if (optss.length === 0)
        return res;
    var more = [];
    if (res === undefined) {
        res = [];
    }
    return Promise.resolve(optss)
        .map(opts => {
            return utils_parse_once(opts, include)
                .map(parsed => {
                    if (parsed.args.length === 0) {
                        res.push(parsed);
                    } else {
                        more.push(parsed);
                    }

		});
	})
        .then(_ => utils_parse_recursive(more, res, include));
}

function utils_parse_once(opts, include) {
    var res = [];
    if (opts.args.length === 0)
        return [opts];

    var el = opts.args[0];
    opts.args = opts.args.slice(1);
    var t;
    switch (el.type) {
        case 'scope':
            var ctxs = utils_parse_scope(opts.ctx, el.arg);
            for (var ctx of ctxs) {
                res.push({
                    'ctx': ctx,
                    'name': opts.name.slice(),
                    'args': opts.args.slice(),
                    'orig': opts.orig.slice(),
                });
            }
            break;
        case 'cond':
	    res = Promise.resolve(utils_apply_template("<%= !!(" + el.arg + ") %>", opts.ctx, {'opts': opts}))
                .then(cond => {
                    if (cond == 'true')
                        return [opts];
	            else
		        return [];
		});
            break;
        case 'includable':
            if (include)
{'fraction_non_alphanumeric': 0.15738108773924578, 'fraction_numerical': 0.0018950161076369148, 'mean_word_length': 1.6480180632212744, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491589', 'n_tokens_mistral': 3281, 'n_tokens_neox': 3194, 'n_words': 674}