<commit_before>var parseScope = require('lexical-scope');
var commondir = require('commondir');
var through = require('through');
var defined = require('defined');

var path = require('path');
var fs = require('fs');

var processModulePath = require.resolve('process/browser.js');
var processModuleSrc = fs.readFileSync(processModulePath, 'utf8');

var bufferModulePath = require.resolve('native-buffer-browserify/bundle.js');
var bufferModuleSrc = fs.readFileSync(bufferModulePath, 'utf8');

var _vars = {
    process: function () {
        return {
            id: processModulePath,
            source: processModuleSrc,
        }
    },
    global: function (row, basedir) {
        return 'typeof self !== "undefined" ? self : '
            + 'typeof window !== "undefined" ? window : {}'
        ;
    },
    Buffer: function () {
        return {
            id: bufferModulePath,
            source: bufferModuleSrc
        }
    },
    __filename: function (row, basedir) {
        var file = '/' + path.relative(basedir, row.id);
        return JSON.stringify(file);
    },
    __dirname: function (row, basedir) {
        var dir = path.dirname('/' + path.relative(basedir, row.id));
        return JSON.stringify(dir);
    }
}

module.exports = function (files, opts) {
    if (!Array.isArray(files)) {
        opts = files;
        files = [];
    }
    if (!opts) opts = {};
    if (!files) files = [];
    var commoner = defined(opts.commondir, commondir);
    if (commoner === false) commoner = function () { return '/' };
    
    var vars = opts.vars || _vars

    var basedir = opts.basedir || (files.length
        ? commoner(files.map(function (x) {
            return path.resolve(path.dirname(x));
        }))
        : '/'
    );
    var varNames = Object.keys(vars);
    
    var quick = RegExp(varNames.map(function (name) {
        return '\\b' + name + '\\b';
    }).join('|'));
    
    var resolved = {};

    return through(write, end);
    
    function write (row) {
        var tr = this;
        
        //remove hashbang if present
        row.source = String(row.source).replace(/^#![^\n]*\n/, '\n');
        
        if (opts.always !== true && !quick.test(row.source)) {
            return tr.queue(row);
        }
        
        var scope = opts.always
            ? { globals: { implicit: varNames } }
            : parseScope(row.source)
        ;
        
        var globals = {};
        
        varNames.forEach(function (name) {
            if (scope.globals.implicit.indexOf(name) >= 0) {
                var value = vars[name].call(tr, row, basedir);
                if (!value) {}
                else if ('object' == typeof value) {
                    value.deps = value.deps || {};
                    if (!resolved[name]) {
                        tr.queue(value);
                        tr.emit('dep', value);
                    }
                    
                    var igName = '__browserify_' + name;
                    row.deps[igName] = value.id;
                    
                    resolved[name] = true;
                    globals[name] = 'require(' 
                        + JSON.stringify(igName)
                        + ')'
                        + (value.suffix || '')
                    ;
                }
                else globals[name] = value;
            }
        });
        
        row.source = closeOver(globals, row.source);
        tr.queue(row);
    }
    
    function end () {
        this.ended = true;
        this.queue(null);
    }
};

module.exports.vars = _vars

function closeOver (globals, src) {
    var keys = Object.keys(globals);
    if (keys.length === 0) return src;
    return 'var ' + keys.map(function (key) {
        return key + '=' + globals[key];
    }).join(',') + ';' + src;
}
<commit_msg>updated to be a regular transform<commit_after>var parseScope = require('lexical-scope');
var commondir = require('commondir');
var through = require('through');
var defined = require('defined');

var path = require('path');
var fs = require('fs');
var processModulePath = require.resolve('process/browser.js');

var defaultVars = {
    process: function () {
        return 'require(' + JSON.stringify(processModulePath) + ')';
    },
    global: function () {
        return 'typeof self !== "undefined" ? self : '
            + 'typeof window !== "undefined" ? window : {}'
        ;
    },
    Buffer: function () {
        return 'require("buffer").Buffer';
    },
    __filename: function (file, basedir) {
        var file = '/' + path.relative(basedir, file);
        return JSON.stringify(file);
    },
    __dirname: function (file, basedir) {
        var dir = path.dirname('/' + path.relative(basedir, file));
        return JSON.stringify(dir);
    }
};

module.exports = function (file, opts) {
    if (!opts) opts = {};
    
    var basedir = opts.basedir || '/';
    var vars = opts.vars || defaultVars
    var varNames = Object.keys(vars);
    
    var quick = RegExp(varNames.map(function (name) {
        return '\\b' + name + '\\b';
    }).join('|'));
    
    var resolved = {};
    var chunks = [];
    
    return through(write, end);
    
    function write (buf) { chunks.push(buf) }
    
    function end () {
        var source = Buffer.concat(chunks).toString('utf8');
        source = source.replace(/^#![^\n]*\n/, '\n');
        
        if (opts.always !== true && !quick.test(source)) {
            this.queue(source);
            this.queue(null);
            return;
        }
        
        var scope = opts.always
            ? { globals: { implicit: varNames } }
            : parseScope(source)
        ;
        
        var globals = {};
        
        varNames.forEach(function (name) {
            if (scope.globals.implicit.indexOf(name) >= 0) {
                var value = vars[name](file, basedir);
                if (value) globals[name] = value;
            }
        });
        
        this.queue(closeOver(globals, source));
        this.queue(null);
    }
};

module.exports.vars = defaultVars;

function closeOver (globals, src) {
    var keys = Object.keys(globals);
    if (keys.length === 0) return src;
    var values = keys.map(function (key) { return globals[key] });
    
    if (keys.length <= 3) {
        return '(function (' + keys.join(',') + '){'
            + src + '}).call(this,' + values.join(',') + ')'
        ;
    }
    // necessary to make arguments[3..6] still work for workerify etc
    // a,b,c,arguments[3..6],d,e,f...
    var extra = [ '__argument', '__argument', '__argument', '__argument' ];
    var names = keys.slice(0,3).concat(extra).concat(keys.slice(3));
    values.splice(3, 0,
        'arguments[3]','arguments[4]',
        'arguments[5]','arguments[6]'
    );
    
    return '(function (' + names.join(',') + '){'
        + src + '}).call(this,' + values.join(',') + ')'
    ;
}

{'fraction_non_alphanumeric': 0.1593677494199536, 'fraction_numerical': 0.003045243619489559, 'mean_word_length': 1.8382716049382717, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373838', 'n_tokens_mistral': 2063, 'n_tokens_neox': 2032, 'n_words': 439}