<commit_before>var fs = require('fs');
var path = require('path');
var spawn = require('child_process').spawn;

var browserResolve = require('browser-resolve');
var nodeResolve = require('resolve');
var detective = require('detective');
var through = require('through2');
var concat = require('concat-stream');
var parents = require('parents');
var combine = require('stream-combiner2');
var duplexer = require('duplexer2');
var copy = require('shallow-copy');

var inherits = require('inherits');
var Transform = require('readable-stream').Transform;

module.exports = Deps;
inherits(Deps, Transform);

function Deps (opts) {
    var self = this;
    if (!(this instanceof Deps)) return new Deps(opts);
    Transform.call(this, { objectMode: true });
    
    if (!opts) opts = {};
    
    this.basedir = opts.basedir || process.cwd();
    this.cache = opts.cache;
    this.pkgCache = opts.packageCache || {};
    this.pkgFileCache = {};
    this.pkgFileCachePending = {};
    this.visited = {};
    this.walking = {};
    this.entries = [];
    
    this.paths = opts.paths || process.env.NODE_PATH;
    if (typeof this.paths === 'string') {
        this.paths = process.env.NODE_PATH.split(':');
    }
    if (!this.paths) this.paths = [];
    
    this.transforms = [].concat(opts.transform).filter(Boolean);
    this.globalTransforms = [].concat(opts.globalTransform).filter(Boolean);
    this.resolver = opts.resolve || browserResolve;
    this.options = opts;
    this.pending = 0;
    
    var topfile = path.join(this.basedir, '__fake.js');
    this.top = { id: topfile, filename: topfile, paths: this.paths };
}

Deps.prototype._transform = function (row, enc, next) {
    var self = this;
    self.pending ++;
    if (typeof row === 'string') {
        row = { file: row };
    }
    if (row.entry !== false) self.entries.push(row.file);
    
    self.lookupPackage(row.file, function (err, pkg) {
        if (err && this.options.ignoreMissing) {
            self.emit('missing', row.file, self.top);
            self.pending --;
            return next();
        }
        if (err) return self.emit('error', err)
        self.pending --;
        start(pkg)
    });
    next();
    
    function start (pkg) {
        if (!pkg) pkg = {};
        if (!pkg.__dirname) pkg.__dirname = path.dirname(row.file);
        self.walk(row, self.top);
    }
};

Deps.prototype._flush = function () {
    if (this.pending === 0) this.push(null);
    this._ended = true;
};

Deps.prototype.resolve = function (id, parent, cb) {
    var self = this;
    var opts = self.options;
    
    if (xhas(self.cache, parent.id, 'deps', id)
    && self.cache[parent.id].deps) {
        var file = self.cache[parent.id].deps[id];
        var pkg = self.pkgCache[file];
        if (pkg) return cb(null, file, pkg);
        return self.lookupPackage(file, function (err, pkg) {
            cb(null, file, pkg);
        });
    }
    
    var pkgdir;
    parent.packageFilter = function (p, x) {
        pkgdir = x;
        if (opts.packageFilter) return opts.packageFilter(p, x);
        else return p;
    };
    
    if (opts.extensions) parent.extensions = opts.extensions;
    if (opts.modules) parent.modules = opts.modules;
    
    self.resolver(id, parent, function onresolve (err, file, pkg) {
        if (err) return cb(err);
        if (!file) return cb(new Error(
            'module not found: "' + id + '" from file '
            + parent.filename
        ));
        
        if (pkg && pkgdir) pkg.__dirname = pkgdir;
        if (!pkg || !pkg.__dirname) {
            self.lookupPackage(file, function (err, p) {
                if (err) return cb(err);
                if (!p) p = {};
                if (!p.__dirname) p.__dirname = path.dirname(file);
                self.pkgCache[file] = p;
                onresolve(err, file, opts.packageFilter
                    ? opts.packageFilter(p, p.__dirname) : p
                );
            });
        }
        else cb(err, file, pkg);
    });
};

Deps.prototype.readFile = function (file, id, pkg) {
    var tr = through();
    if (this.cache && this.cache[file]) {
        tr.push(this.cache[file].source);
        tr.push(null);
        return tr;
    }
    var rs = fs.createReadStream(file);
    rs.on('error', function (err) { tr.emit('error', err) });
    this.emit('file', file, id);
    return rs.pipe(this.getTransforms(file, pkg));
};

Deps.prototype.getTransforms = function (file, pkg) {
    var self = this;
    var isTopLevel = this.entries.some(function (main) {
        var m = path.relative(path.dirname(main), file);
        return m.split(/[\\\/]/).indexOf('node_modules') < 0;
    });
    
    var transforms = [].concat(isTopLevel ? this.transforms : [])
        .concat(getTransforms(pkg, {
            globalTransform: this.globalTransforms,
            transformKey: this.options.transformKey
        }))
    ;
    if (transforms.length === 0) return through();
    
    var pending = transforms.length;
    var streams = [];
    var input = through();
    var output = through();
    var dup = duplexer(input, output);
    
    for (var i = 0; i < transforms.length; i++) (function (i) {
        makeTransform(transforms[i], function (err, trs) {
            if (err) return self.emit('error', err)
            streams[i] = trs;
            if (-- pending === 0) done();
        });
    })(i);
    return dup;
    
    function done () {
        var middle = combine.apply(null, streams);
        middle.on('error', function (err) {
            err.message += ' while parsing file: ' + file;
            if (!err.filename) err.filename = file;
            self.emit('error', err);
        });
        input.pipe(middle).pipe(output);
    }
    
    function makeTransform (tr, cb) {
        var trOpts = {};
        if (Array.isArray(tr)) {
            trOpts = tr[1];
            tr = tr[0];
        }
        if (typeof tr === 'function') {
            var t = tr(file, trOpts);
            self.emit('transform', t, file);
            nextTick(cb, null, wrapTransform(t));
        }
        else {
            loadTransform(tr, trOpts, function (err, trs) {
                if (err) return cb(err);
                cb(null, wrapTransform(trs));
            });
        }
    }
    
    function loadTransform (id, trOpts, cb) {
        var params = {
            basedir: trOpts.basedir
                || self.options.basedir
                || path.dirname(file)
        };
        nodeResolve(id, params, function nr (err, res, again) {
            if (err && again) return cb(err);
            
            if (err) {
                params.basedir = pkg.__dirname;
                return nodeResolve(id, params, function (e, r) {
                    nr(e, r, true)
                });
            }
            
            if (!res) return cb(new Error(
                'cannot find transform module ' + tr
                + ' while transforming ' + file
            ));
            
            var r = require(res);
            if (typeof r !== 'function') {
                return cb(new Error('transform not a function'));
            }
            
            var trs = r(file, trOpts);
            self.emit('transform', trs, file);
            cb(null, trs);
        });
    }
};

Deps.prototype.walk = function (id, parent, cb) {
    var self = this;
    var opts = self.options;
    this.pending ++;
    
    var rec = {};
    if (typeof id === 'object') {
        rec = copy(id);
        if (rec.entry === false) delete rec.entry;
        id = rec.file || rec.id;
    }
    
    self.resolve(id, parent, function (err, file, pkg) {
        if (rec.expose) {
            self.options.modules[rec.expose] = file;
        }
        if (pkg) self.emit('package', pkg);
        
        if (opts.postFilter && !opts.postFilter(id, file, pkg)) {
            if (--self.pending === 0) self.push(null);
            return cb(null, undefined);
        }
        if (err && rec.source) {
            file = rec.file;
            
            var ts = self.getTransforms(file, pkg);
            ts.pipe(concat(function (body) {
                rec.source = body.toString('utf8');
                fromSource(rec.source);
            }));
            return ts.end(rec.source);
        }
        if (err && self.options.ignoreMissing) {
            if (--self.pending === 0) self.push(null);
            self.emit('missing', id, parent);
            return cb(null, undefined);
        }
        if (err) return self.emit('error', err);
        if (self.visited[file]) {
            if (-- self.pending === 0) self.push(null);
            return cb && cb(null, file);
        }
        self.visited[file] = true;
        
        if (rec.source) {
            var ts = self.getTransforms(file, pkg);
            ts.pipe(concat(function (body) {
                rec.source = body.toString('utf8');
                fromSource(rec.source);
            }));
            return ts.end(rec.source);
        }
        
        var c = self.cache && self.cache[file];
        if (c) return fromDeps(file, c.source, c.package, Object.keys(c.deps));
        
        self.readFile(file, id, pkg).pipe(concat(function (body) {
            fromSource(body.toString('utf8'));
        }));
        
        function fromSource (src) {
            var deps = rec.noparse ? [] : self.parseDeps(file, src);
            if (deps) fromDeps(file, src, pkg, deps);
        }
    });
    
    function fromDeps (file, src, pkg, deps) {
        var p = deps.length;
        var current = {
            id: file,
            filename: file,
            paths: self.paths,
            package: pkg
        };
        var resolved = {};
        
        deps.forEach(function (id) {
            if (opts.filter && !opts.filter(id)) {
                resolved[id] = false;
                if (--p === 0) done();
                return;
            }
            self.walk(id, current, function (err, r) {
                resolved[id] = r;
                if (--p === 0) done();
            });
        });
        if (deps.length === 0) done();
        
        function done () {
            if (!rec.id) rec.id = file;
            if (!rec.source) rec.source = src;
            if (!rec.deps) rec.deps = resolved;
            if (!rec.file) rec.file = file;
            
            if (self.entries.indexOf(file) >= 0) {
                rec.entry = true;
            }
            self.push(rec);
            
            if (cb) cb(null, file);
            if (-- self.pending === 0 && self._ended) self.push(null);
        }
    }
};

Deps.prototype.parseDeps = function (file, src, cb) {
    if (this.options.noParse === true) return [];
    if (/\.json$/.test(file)) return [];
    
    if (Array.isArray(this.options.noParse)
    && this.options.noParse.indexOf(file) >= 0) {
        return [];
    }
    
    try { var deps = detective(src) }
    catch (ex) {
        var message = ex && ex.message ? ex.message : ex;
        this.emit('error', new Error(
            'Parsing file ' + file + ': ' + message
        ));
        return;
    }
    return deps;
};

Deps.prototype.lookupPackage = function (file, cb) {
    var self = this;
    
    var cached = this.pkgCache[file];
    if (cached) return nextTick(cb, null, cached);
    if (cached === false) return nextTick(cb, null, undefined);
    
    var dirs = parents(path.dirname(file));
    
    (function next () {
        if (dirs.length === 0) {
            self.pkgCache[file] = false;
            return cb(null, undefined);
        }
        var dir = dirs.shift();
        if (dir.split(/[\\\/]/).slice(-1)[0] === 'node_modules') {
            return cb(null, undefined);
        }
        
        var pkgfile = path.join(dir, 'package.json');
        
        var cached = self.pkgCache[pkgfile];
        if (cached) return nextTick(cb, null, cached);
        else if (cached === false) return next();
        
        var pcached = self.pkgFileCachePending[pkgfile];
        if (pcached) return pcached.push(onpkg);
        pcached = self.pkgFileCachePending[pkgfile] = [];
        
        fs.readFile(pkgfile, function (err, src) {
            if (err) return onpkg();
            try { var pkg = JSON.parse(src) }
            catch (err) {
                return onpkg(new Error([
                    err + ' while parsing json file ' + pkgfile
                ].join('')))
            }
            pkg.__dirname = dir;
            
            self.pkgCache[pkgfile] = pkg;
            self.pkgCache[file] = pkg;
            onpkg(null, pkg);
        });
        
        function onpkg (err, pkg) {
            if (self.pkgFileCachePending[pkgfile]) {
                var fns = self.pkgFileCachePending[pkgfile];
                delete self.pkgFileCachePending[pkgfile];
                fns.forEach(function (f) { f(err, pkg) });
            }
            if (err) cb(err)
            else if (pkg) cb(null, pkg)
            else {
                self.pkgCache[pkgfile] = false;
                next();
            }
        }
    })();
};
 
function getTransforms (pkg, opts) {
    var trx = [];
    if (opts.transformKey) {
        var n = pkg;
        var keys = opts.transformKey;
        for (var i = 0; i < keys.length; i++) {
            if (n && typeof n === 'object') n = n[keys[i]];
            else break;
        }
        if (i === keys.length) {
            trx = [].concat(n).filter(Boolean);
        }
    }
    return trx.concat(opts.globalTransform || []);
}

function nextTick (cb) {
    var args = [].slice.call(arguments, 1);
    process.nextTick(function () { cb.apply(null, args) });
}

function xhas (obj) {
    if (!obj) return false;
    for (var i = 1; i < arguments.length; i++) {
        var key = arguments[i];
        if (!has(obj, key)) return false;
        obj = obj[key];
    }
    return true;
}

function has (obj, key) {
    return Object.prototype.hasOwnProperty.call(obj, key);
}

function wrapTransform (tr) {
    if (typeof tr.read === 'function') return tr;
    var input = through(), output = through();
    input.pipe(tr).pipe(output);
    var wrapper = duplexer(input, output);
    tr.on('error', function (err) { wrapper.emit('error', err) });
    return wrapper;
}
<commit_msg>actually that was a mistake<commit_after>var fs = require('fs');
var path = require('path');
var spawn = require('child_process').spawn;

var browserResolve = require('browser-resolve');
var nodeResolve = require('resolve');
var detective = require('detective');
var through = require('through2');
var concat = require('concat-stream');
var parents = require('parents');
var combine = require('stream-combiner2');
var duplexer = require('duplexer2');
var copy = require('shallow-copy');

var inherits = require('inherits');
var Transform = require('readable-stream').Transform;

module.exports = Deps;
inherits(Deps, Transform);

function Deps (opts) {
    var self = this;
    if (!(this instanceof Deps)) return new Deps(opts);
    Transform.call(this, { objectMode: true });
    
    if (!opts) opts = {};
    
    this.basedir = opts.basedir || process.cwd();
    this.cache = opts.cache;
    this.pkgCache = opts.packageCache || {};
    this.pkgFileCache = {};
    this.pkgFileCachePending = {};
    this.visited = {};
    this.walking = {};
    this.entries = [];
    
    this.paths = opts.paths || process.env.NODE_PATH;
    if (typeof this.paths === 'string') {
        this.paths = process.env.NODE_PATH.split(':');
    }
    if (!this.paths) this.paths = [];
    
    this.transforms = [].concat(opts.transform).filter(Boolean);
    this.globalTransforms = [].concat(opts.globalTransform).filter(Boolean);
    this.resolver = opts.resolve || browserResolve;
    this.options = opts;
    this.pending = 0;
    
    var topfile = path.join(this.basedir, '__fake.js');
    this.top = { id: topfile, filename: topfile, paths: this.paths };
}

Deps.prototype._transform = function (row, enc, next) {
    var self = this;
    self.pending ++;
    if (typeof row === 'string') {
        row = { file: row };
    }
    if (row.entry !== false) self.entries.push(row.file);
    
    self.lookupPackage(row.file, function (err, pkg) {
        if (err && this.options.ignoreMissing) {
            self.emit('missing', row.file, self.top);
            self.pending --;
            return next();
        }
        if (err) return self.emit('error', err)
        self.pending --;
        start(pkg)
    });
    next();
    
    function start (pkg) {
        if (!pkg) pkg = {};
        if (!pkg.__dirname) pkg.__dirname = path.dirname(row.file);
        self.walk(row, self.top);
    }
};

Deps.prototype._flush = function () {
    if (this.pending === 0) this.push(null);
    this._ended = true;
};

Deps.prototype.resolve = function (id, parent, cb) {
    var self = this;
    var opts = self.options;
    
    if (xhas(self.cache, parent.id, 'deps', id)
    && self.cache[parent.id].deps) {
        var file = self.cache[parent.id].deps[id];
        var pkg = self.pkgCache[file];
        if (pkg) return cb(null, file, pkg);
        return self.lookupPackage(file, function (err, pkg) {
            cb(null, file, pkg);
        });
    }
    
    var pkgdir;
    parent.packageFilter = function (p, x) {
        pkgdir = x;
        if (opts.packageFilter) return opts.packageFilter(p, x);
        else return p;
    };
    
    if (opts.extensions) parent.extensions = opts.extensions;
    if (opts.modules) parent.modules = opts.modules;
    
    self.resolver(id, parent, function onresolve (err, file, pkg) {
        if (err) return cb(err);
        if (!file) return cb(new Error(
            'module not found: "' + id + '" from file '
            + parent.filename
        ));
        
        if (pkg && pkgdir) pkg.__dirname = pkgdir;
        if (!pkg || !pkg.__dirname) {
            self.lookupPackage(file, function (err, p) {
                if (err) return cb(err);
                if (!p) p = {};
                if (!p.__dirname) p.__dirname = path.dirname(file);
                self.pkgCache[file] = p;
                onresolve(err, file, opts.packageFilter
                    ? opts.packageFilter(p, p.__dirname) : p
                );
            });
        }
        else cb(err, file, pkg);
    });
};

Deps.prototype.readFile = function (file, id, pkg) {
    var tr = through();
    if (this.cache && this.cache[file]) {
        tr.push(this.cache[file].source);
        tr.push(null);
        return tr;
    }
    var rs = fs.createReadStream(file);
    rs.on('error', function (err) { tr.emit('error', err) });
    this.emit('file', file, id);
    return rs.pipe(this.getTransforms(file, pkg));
};

Deps.prototype.getTransforms = function (file, pkg) {
    var self = this;
    var isTopLevel = this.entries.some(function (main) {
        var m = path.relative(path.dirname(main), file);
        return m.split(/[\\\/]/).indexOf('node_modules') < 0;
    });
    
    var transforms = [].concat(isTopLevel ? this.transforms : [])
        .concat(getTransforms(pkg, {
            globalTransform: this.globalTransforms,
            transformKey: this.options.transformKey
        }))
    ;
    if (transforms.length === 0) return through();
    
    var pending = transforms.length;
    var streams = [];
    var input = through();
    var output = through();
    var dup = duplexer(input, output);
    
    for (var i = 0; i < transforms.length; i++) (function (i) {
        makeTransform(transforms[i], function (err, trs) {
            if (err) return self.emit('error', err)
            streams[i] = trs;
            if (-- pending === 0) done();
        });
    })(i);
    return dup;
    
    function done () {
        var middle = combine.apply(null, streams);
        middle.on('error', function (err) {
            err.message += ' while parsing file: ' + file;
            if (!err.filename) err.filename = file;
            self.emit('error', err);
        });
        input.pipe(middle).pipe(output);
    }
    
    function makeTransform (tr, cb) {
        var trOpts = {};
        if (Array.isArray(tr)) {
            trOpts = tr[1];
            tr = tr[0];
        }
        if (typeof tr === 'function') {
            var t = tr(file, trOpts);
            self.emit('transform', t, file);
            nextTick(cb, null, wrapTransform(t));
        }
        else {
            loadTransform(tr, trOpts, function (err, trs) {
                if (err) return cb(err);
                cb(null, wrapTransform(trs));
            });
        }
    }
    
    function loadTransform (id, trOpts, cb) {
        var params = { basedir: path.dirname(file) };
        nodeResolve(id, params, function nr (err, res, again) {
            if (err && again) return cb(err);
            
            if (err) {
                params.basedir = pkg.__dirname;
                return nodeResolve(id, params, function (e, r) {
                    nr(e, r, true)
                });
            }
            
            if (!res) return cb(new Error(
                'cannot find transform module ' + tr
                + ' while transforming ' + file
            ));
            
            var r = require(res);
            if (typeof r !== 'function') {
                return cb(new Error('transform not a function'));
            }
            
            var trs = r(file, trOpts);
            self.emit('transform', trs, file);
            cb(null, trs);
        });
    }
};

Deps.prototype.walk = function (id, parent, cb) {
    var self = this;
    var opts = self.options;
    this.pending ++;
    
    var rec = {};
    if (typeof id === 'object') {
        rec = copy(id);
        if (rec.entry === false) delete rec.entry;
        id = rec.file || rec.id;
    }
    
    self.resolve(id, parent, function (err, file, pkg) {
        if (rec.expose) {
            self.options.modules[rec.expose] = file;
        }
        if (pkg) self.emit('package', pkg);
        
        if (opts.postFilter && !opts.postFilter(id, file, pkg)) {
            if (--self.pending === 0) self.push(null);
            return cb(null, undefined);
        }
        if (err && rec.source) {
            file = rec.file;
            
            var ts = self.getTransforms(file, pkg);
            ts.pipe(concat(function (body) {
                rec.source = body.toString('utf8');
                fromSource(rec.source);
            }));
            return ts.end(rec.source);
        }
        if (err && self.options.ignoreMissing) {
            if (--self.pending === 0) self.push(null);
            self.emit('missing', id, parent);
            return cb(null, undefined);
        }
        if (err) return self.emit('error', err);
        if (self.visited[file]) {
            if (-- self.pending === 0) self.push(null);
            return cb && cb(null, file);
        }
        self.visited[file] = true;
        
        if (rec.source) {
            var ts = self.getTransforms(file, pkg);
            ts.pipe(concat(function (body) {
                rec.source = body.toString('utf8');
                fromSource(rec.source);
            }));
            return ts.end(rec.source);
        }
        
        var c = self.cache && self.cache[file];
        if (c) return fromDeps(file, c.source, c.package, Object.keys(c.deps));
        
        self.readFile(file, id, pkg).pipe(concat(function (body) {
            fromSource(body.toString('utf8'));
        }));
        
        function fromSource (src) {
            var deps = rec.noparse ? [] : self.parseDeps(file, src);
            if (deps) fromDeps(file, src, pkg, deps);
        }
    });
    
    function fromDeps (file, src, pkg, deps) {
        var p = deps.length;
        var current = {
            id: file,
            filename: file,
            paths: self.paths,
            package: pkg
        };
        var resolved = {};
        
        deps.forEach(function (id) {
            if (opts.filter && !opts.filter(id)) {
                resolved[id] = false;
                if (--p === 0) done();
                return;
            }
            self.walk(id, current, function (err, r) {
                resolved[id] = r;
                if (--p === 0) done();
            });
        });
        if (deps.length === 0) done();
        
        function done () {
            if (!rec.id) rec.id = file;
            if (!rec.source) rec.source = src;
            if (!rec.deps) rec.deps = resolved;
            if (!rec.file) rec.file = file;
            
            if (self.entries.indexOf(file) >= 0) {
                rec.entry = true;
            }
            self.push(rec);
            
            if (cb) cb(null, file);
            if (-- self.pending === 0 && self._ended) self.push(null);
        }
    }
};

Deps.prototype.parseDeps = function (file, src, cb) {
    if (this.options.noParse === true) return [];
    if (/\.json$/.test(file)) return [];
    
    if (Array.isArray(this.options.noParse)
    && this.options.noParse.indexOf(file) >= 0) {
        return [];
    }
    
    try { var deps = detective(src) }
    catch (ex) {
        var message = ex && ex.message ? ex.message : ex;
        this.emit('error', new Error(
            'Parsing file ' + file + ': ' + message
        ));
        return;
    }
    return deps;
};

Deps.prototype.lookupPackage = function (file, cb) {
    var self = this;
    
    var cached = this.pkgCache[file];
    if (cached) return nextTick(cb, null, cached);
    if (cached === false) return nextTick(cb, null, undefined);
    
    var dirs = parents(path.dirname(file));
    
    (function next () {
        if (dirs.length === 0) {
            self.pkgCache[file] = false;
            return cb(null, undefined);
        }
        var dir = dirs.shift();
        if (dir.split(/[\\\/]/).slice(-1)[0] === 'node_modules') {
            return cb(null, undefined);
        }
        
        var pkgfile = path.join(dir, 'package.json');
        
        var cached = self.pkgCache[pkgfile];
        if (cached) return nextTick(cb, null, cached);
        else if (cached === false) return next();
        
        var pcached = self.pkgFileCachePending[pkgfile];
        if (pcached) return pcached.push(onpkg);
        pcached = self.pkgFileCachePending[pkgfile] = [];
        
        fs.readFile(pkgfile, function (err, src) {
            if (err) return onpkg();
            try { var pkg = JSON.parse(src) }
            catch (err) {
                return onpkg(new Error([
                    err + ' while parsing json file ' + pkgfile
                ].join('')))
            }
            pkg.__dirname = dir;
            
            self.pkgCache[pkgfile] = pkg;
            self.pkgCache[file] = pkg;
            onpkg(null, pkg);
        });
        
        function onpkg (err, pkg) {
            if (self.pkgFileCachePending[pkgfile]) {
                var fns = self.pkgFileCachePending[pkgfile];
                delete self.pkgFileCachePending[pkgfile];
                fns.forEach(function (f) { f(err, pkg) });
            }
            if (err) cb(err)
            else if (pkg) cb(null, pkg)
            else {
                self.pkgCache[pkgfile] = false;
                next();
            }
        }
    })();
};
 
function getTransforms (pkg, opts) {
    var trx = [];
    if (opts.transformKey) {
        var n = pkg;
        var keys = opts.transformKey;
        for (var i = 0; i < keys.length; i++) {
            if (n && typeof n === 'object') n = n[keys[i]];
            else break;
        }
        if (i === keys.length) {
            trx = [].concat(n).filter(Boolean);
        }
    }
    return trx.concat(opts.globalTransform || []);
}

function nextTick (cb) {
    var args = [].slice.call(arguments, 1);
    process.nextTick(function () { cb.apply(null, args) });
}

function xhas (obj) {
    if (!obj) return false;
    for (var i = 1; i < arguments.length; i++) {
        var key = arguments[i];
        if (!has(obj, key)) return false;
        obj = obj[key];
    }
    return true;
}

function has (obj, key) {
    return Object.prototype.hasOwnProperty.call(obj, key);
}

function wrapTransform (tr) {
    if (typeof tr.read === 'function') return tr;
    var input = through(), output = through();
    input.pipe(tr).pipe(output);
    var wrapper = duplexer(input, output);
    tr.on('error', function (err) { wrapper.emit('error', err) });
    return wrapper;
}

{'fraction_non_alphanumeric': 0.14978917779339423, 'fraction_numerical': 0.002037947997189037, 'mean_word_length': 1.7371609924985574, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2413822', 'n_tokens_mistral': 8721, 'n_tokens_neox': 8548, 'n_words': 2054}