<commit_before>var fs = require('fs');
var path = require('path');

var through = require('through2');
var Readable = require('readable-stream').Readable;

var concat = require('concat-stream');
var duplexer = require('duplexer2');
var falafel = require('falafel');
var unparse = require('escodegen').generate;
var inspect = require('object-inspect');
var evaluate = require('static-eval');
var copy = require('shallow-copy');

module.exports = function (modules, opts) {
    var varNames = {};
    if (!opts) opts = {};
    var vars = opts.vars || {};
    var pending = 0;
    var updates = [];
    
    function pushUpdate (node, s) {
        var rep = String(s);
        var prev = node.range[1] - node.range[0];
        updates.push({ offset: prev - rep.length });
        node.update(rep);
    }
    
    var output = through();
    return duplexer(concat(function (body) {
        var src = falafel(body.toString('utf8'), walk);
        //try { var src = falafel(body.toString('utf8'), walk) }
        //catch (err) { return error(err) }
        if (pending === 0) finish(src);
    }), output);
    
    function finish (src) {
        var offset = 0, pos = 0;
        src = String(src);
        
        (function next () {
            if (updates.length === 0) return done();
            
            var s = updates.shift();
            if (!s.stream) {
                offset += s.offset;
                return next();
            }
            
            output.push(src.slice(pos, s.range[0] - offset));
            pos = s.range[0] - offset;
            offset += s.range[1] - s.range[0];
            
            s.stream.on('end', next);
            s.stream.pipe(output, { end: false });
        })();
        
        function done () {
            output.push(src.slice(pos));
            output.push(null);
        }
    }
    
    function error (msg) {
        output.emit('error', new Error(msg));
    }
    
    function walk (node) {
        var isreq = false, reqid;
        if (isRequire(node)) {
            reqid = node.arguments[0].value;
            isreq = has(modules, reqid);
        }
        
        if (isreq && node.parent.type === 'VariableDeclarator'
        && node.parent.id.type === 'Identifier') {
            varNames[node.parent.id.name] = reqid;
            var decs = node.parent.parent.declarations;
            var ix = decs.indexOf(node.parent);
            if (ix >= 0) decs.splice(ix, 1);
            
            if (decs.length === 0) {
                pushUpdate(node.parent.parent, '');
            }
            else {
                pushUpdate(
                    node.parent.parent,
                    unparse(node.parent.parent)
                );
            }
        }
        else if (isreq && node.parent.type === 'AssignmentExpression'
        && node.parent.left.type === 'Identifier') {
            varNames[node.parent.left.name] = reqid;
            var cur = node.parent.parent;
            if (cur.type === 'SequenceExpression') {
                var ex = cur.expressions;
                var ix = ex.indexOf(node.parent);
                if (ix >= 0) ex.splice(ix, 1);
                pushUpdate(
                    node.parent.parent,
                    unparse(node.parent.parent)
                );
            }
            else pushUpdate(cur, '');
        }
        else if (isreq && node.parent.type === 'MemberExpression'
        && node.parent.property.type === 'Identifier'
        && node.parent.parent.type === 'VariableDeclarator'
        && node.parent.parent.id.type === 'Identifier') {
            varNames[node.parent.parent.id.name] = [
                reqid, node.parent.property.name
            ];
            var decs = node.parent.parent.parent.declarations;
            var ix = decs.indexOf(node.parent.parent);
            if (ix >= 0) decs.splice(ix, 1);
            
            if (decs.length === 0) {
                pushUpdate(node.parent.parent.parent, '');
            }
            else {
                pushUpdate(
                    node.parent.parent.parent,
                    unparse(node.parent.parent.parent)
                );
            }
        }
        else if (isreq && node.parent.type === 'MemberExpression'
        && node.parent.property.type === 'Identifier') {
            //console.log('REQUIRE!!!');
            if (node.parent.parent.type === 'CallExpression') {
                pushUpdate(node.parent.parent, '');
            }
            else {
                pushUpdate(node.parent, '');
            }
            var name = node.parent.property.name;
            var cur = copy(node.parent.parent);
            cur.callee = copy(node.parent.property);
            cur.callee.parent = cur;
            traverse(cur.callee, modules[reqid][name]);
        }
        
        if (node.type === 'Identifier' && varNames[node.name]) {
            var vn = varNames[node.name];
            if (Array.isArray(vn)) {
                traverse(node, modules[vn[0]][vn[1]]);
            }
            else traverse(node, modules[vn]);
        }
    }
    
    function traverse (node, val) {
        if (node.parent.type === 'CallExpression') {
            if (typeof val !== 'function') {
                return error(
                    'tried to statically call ' + inspect(val)
                    + ' as a function'
                );
            }
            var xvars = copy(vars);
            xvars[node.name] = val;
            var res = evaluate(node.parent, xvars);
            
            if (isStream(res)) {
                updates.push({<commit_msg>all inline tests now passing<commit_after>var fs = require('fs');
var path = require('path');

var through = require('through2');
var Readable = require('readable-stream').Readable;

var concat = require('concat-stream');
var duplexer = require('duplexer2');
var falafel = require('falafel');
var unparse = require('escodegen').generate;
var inspect = require('object-inspect');
var evaluate = require('static-eval');
var copy = require('shallow-copy');

module.exports = function (modules, opts) {
    var varNames = {};
    if (!opts) opts = {};
    var vars = opts.vars || {};
    var pending = 0;
    var updates = [];
    
    function pushUpdate (node, s) {
        var rep = String(s);
        var prev = node.range[1] - node.range[0];
        updates.push({ offset: prev - rep.length });
        node.update(rep);
    }
    
    var output = through();
    return duplexer(concat(function (body) {
        var src = falafel(body.toString('utf8'), walk);
        //try { var src = falafel(body.toString('utf8'), walk) }
        //catch (err) { return error(err) }
        if (pending === 0) finish(src);
    }), output);
    
    function finish (src) {
        var offset = 0, pos = 0;
        src = String(src);
        
        (function next () {
            if (updates.length === 0) return done();
            
            var s = updates.shift();
            if (!s.stream) {
                offset += s.offset;
                return next();
            }
            
            output.push(src.slice(pos, s.range[0] - offset));
            pos = s.range[0] - offset;
            offset += s.range[1] - s.range[0];
            
            s.stream.on('end', next);
            s.stream.pipe(output, { end: false });
        })();
        
        function done () {
            output.push(src.slice(pos));
            output.push(null);
        }
    }
    
    function error (msg) {
        output.emit('error', new Error(msg));
    }
    
    function walk (node) {
        var isreq = false, reqid;
        if (isRequire(node)) {
            reqid = node.arguments[0].value;
            isreq = has(modules, reqid);
        }
        
        if (isreq && node.parent.type === 'VariableDeclarator'
        && node.parent.id.type === 'Identifier') {
            varNames[node.parent.id.name] = reqid;
            var decs = node.parent.parent.declarations;
            var ix = decs.indexOf(node.parent);
            if (ix >= 0) decs.splice(ix, 1);
            
            if (decs.length === 0) {
                pushUpdate(node.parent.parent, '');
            }
            else {
                pushUpdate(
                    node.parent.parent,
                    unparse(node.parent.parent)
                );
            }
        }
        else if (isreq && node.parent.type === 'AssignmentExpression'
        && node.parent.left.type === 'Identifier') {
            varNames[node.parent.left.name] = reqid;
            var cur = node.parent.parent;
            if (cur.type === 'SequenceExpression') {
                var ex = cur.expressions;
                var ix = ex.indexOf(node.parent);
                if (ix >= 0) ex.splice(ix, 1);
                pushUpdate(
                    node.parent.parent,
                    unparse(node.parent.parent)
                );
            }
            else pushUpdate(cur, '');
        }
        else if (isreq && node.parent.type === 'MemberExpression'
        && node.parent.property.type === 'Identifier'
        && node.parent.parent.type === 'VariableDeclarator'
        && node.parent.parent.id.type === 'Identifier') {
            varNames[node.parent.parent.id.name] = [
                reqid, node.parent.property.name
            ];
            var decs = node.parent.parent.parent.declarations;
            var ix = decs.indexOf(node.parent.parent);
            if (ix >= 0) decs.splice(ix, 1);
            
            if (decs.length === 0) {
                pushUpdate(node.parent.parent.parent, '');
            }
            else {
                pushUpdate(
                    node.parent.parent.parent,
                    unparse(node.parent.parent.parent)
                );
            }
        }
        else if (isreq && node.parent.type === 'MemberExpression'
        && node.parent.property.type === 'Identifier') {
            if (node.parent.parent.type === 'CallExpression') {
                pushUpdate(node.parent.parent, '');
            }
            else {
                pushUpdate(node.parent, '');
            }
            var name = node.parent.property.name;
            var cur = copy(node.parent.parent);
            cur.callee = copy(node.parent.property);
            cur.callee.parent = cur;
            traverse(cur.callee, modules[reqid][name]);
        }
        else if (isreq && node.parent.type === 'CallExpression') {
            var cur = copy(node.parent);
            var iname = Math.pow(16,8) * Math.random();
            cur.callee = {
                type: 'Identifier',
                name: '_' + Math.floor(iname).toString(16),
                parent: cur
            };
            pushUpdate(node.parent, '');
            traverse(cur.callee, modules[reqid]);
        }
        
        if (node.type === 'Identifier' && varNames[node.name]) {
            var vn = varNames[node.name];
            if (Array.isArray(vn)) {
                traverse(node, modules[vn[0]][vn[1]]);
            }
            else traverse(node, modules[vn]);
        }
    }
    
    function traverse (node, val) {
        if (node.parent.type === 'CallExpression') {
            if (typeof val !== 'function') {
                return error(
                    'tried to statically call ' + inspect(val)
                    + ' as a function'
                );
            }
            var xvars = copy(vars);
            xvars[node.name] = val;
            var res = evaluate(node.parent, xvars);
            
            if (isStream(res)) {
                updates.push({
{'fraction_non_alphanumeric': 0.1418854928971158, 'fraction_numerical': 0.004907447266465777, 'mean_word_length': 1.4610169491525424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125673', 'n_tokens_mistral': 3334, 'n_tokens_neox': 3233, 'n_words': 660}