<commit_before>var parseScope = require('lexical-scope');
var through = require('through2');
var merge = require('xtend');

var path = require('path');
var processPath = require.resolve('process/browser.js');
var isbufferPath = require.resolve('is-buffer')
var combineSourceMap = require('combine-source-map');

var defaultVars = {
    process: function () {
        return 'require(' + JSON.stringify(processPath) + ')';
    },
    global: function () {
        return 'typeof global !== "undefined" ? global : '
            + 'typeof self !== "undefined" ? self : '
            + 'typeof window !== "undefined" ? window : {}'
        ;
    },
    'Buffer.isBuffer': function () {
        return 'require(' + JSON.stringify(isbufferPath) + ')';
    },
    Buffer: function () {
        return 'require("buffer").Buffer';
    },
    __filename: function (file, basedir) {
        var filename = '/' + path.relative(basedir, file);
        return JSON.stringify(filename);
    },
    __dirname: function (file, basedir) {
        var dir = path.dirname('/' + path.relative(basedir, file));
        return JSON.stringify(dir);
    }
};

module.exports = function (file, opts) {
    if (/\.json$/i.test(file)) return through();
    if (!opts) opts = {};
    
    var basedir = opts.basedir || '/';
    var vars = merge(defaultVars, opts.vars);
    var varNames = Object.keys(vars).filter(function(name) {
        return typeof vars[name] === 'function';
    });
    
    var quick = RegExp(varNames.map(function (name) {
        return '\\b' + name + '\\b';
    }).join('|'));
    
    var chunks = [];
    
    return through(write, end);
    
    function write (chunk, enc, next) { chunks.push(chunk); next() }
    
    function end () {
        var self = this;
        var source = Buffer.isBuffer(chunks[0])
            ? Buffer.concat(chunks).toString('utf8')
            : chunks.join('')
        ;
        source = source
            .replace(/^\ufeff/, '')
            .replace(/^#![^\n]*\n/, '\n');
        
        if (opts.always !== true && !quick.test(source)) {
            this.push(source);
            this.push(null);
            return;
        }
        
        try {
            var scope = opts.always
                ? { globals: { implicit: varNames } }
                : parseScope('(function(){\n' + source + '\n})()')
            ;
        }
        catch (err) {
            var e = new SyntaxError(
                (err.message || err) + ' while parsing ' + file
            );
            e.type = 'syntax';
            e.filename = file;
            return this.emit('error', e);
        }
        
        var globals = {};
        
        varNames.forEach(function (name) {
            var match = false;
            if (/\./.test(name)) {
                var parts = name.split('.')
                var prop = scope.globals.implicitProperties[parts[0]]
                if (prop && prop.length === 1 && prop[0] === parts[1]) {
                    match = true;
                }
            }
            else if (scope.globals.implicit.indexOf(name) >= 0) {
                match = true;
            }
            if (match) {
                var value = vars[name](file, basedir);
                if (value) {
                    globals[name] = value;
                    self.emit('global', name);
                }
            }
        });
        
        this.push(closeOver(globals, source, file, opts));
        this.push(null);
    }
};

module.exports.vars = defaultVars;

function closeOver (globals, src, file, opts) {
    var keys = Object.keys(globals);
    if (keys.length === 0) return src;
    var values = keys.map(function (key) { return globals[key] });
    
    var wrappedSource;
    if (keys.length <= 3) {
        wrappedSource = '(function (' + keys.join(',') + '){\n'
            + src + '\n}).call(this,' + values.join(',') + ')'<commit_msg>split dot parsing passes all tests<commit_after>var parseScope = require('lexical-scope');
var through = require('through2');
var merge = require('xtend');

var path = require('path');
var processPath = require.resolve('process/browser.js');
var isbufferPath = require.resolve('is-buffer')
var combineSourceMap = require('combine-source-map');

var defaultVars = {
    process: function () {
        return 'require(' + JSON.stringify(processPath) + ')';
    },
    global: function () {
        return 'typeof global !== "undefined" ? global : '
            + 'typeof self !== "undefined" ? self : '
            + 'typeof window !== "undefined" ? window : {}'
        ;
    },
    'Buffer.isBuffer': function () {
        return 'require(' + JSON.stringify(isbufferPath) + ')';
    },
    Buffer: function () {
        return 'require("buffer").Buffer';
    },
    __filename: function (file, basedir) {
        var filename = '/' + path.relative(basedir, file);
        return JSON.stringify(filename);
    },
    __dirname: function (file, basedir) {
        var dir = path.dirname('/' + path.relative(basedir, file));
        return JSON.stringify(dir);
    }
};

module.exports = function (file, opts) {
    if (/\.json$/i.test(file)) return through();
    if (!opts) opts = {};
    
    var basedir = opts.basedir || '/';
    var vars = merge(defaultVars, opts.vars);
    var varNames = Object.keys(vars).filter(function(name) {
        return typeof vars[name] === 'function';
    });
    
    var quick = RegExp(varNames.map(function (name) {
        return '\\b' + name + '\\b';
    }).join('|'));
    
    var chunks = [];
    
    return through(write, end);
    
    function write (chunk, enc, next) { chunks.push(chunk); next() }
    
    function end () {
        var self = this;
        var source = Buffer.isBuffer(chunks[0])
            ? Buffer.concat(chunks).toString('utf8')
            : chunks.join('')
        ;
        source = source
            .replace(/^\ufeff/, '')
            .replace(/^#![^\n]*\n/, '\n');
        
        if (opts.always !== true && !quick.test(source)) {
            this.push(source);
            this.push(null);
            return;
        }
        
        try {
            var scope = opts.always
                ? { globals: { implicit: varNames } }
                : parseScope('(function(){\n' + source + '\n})()')
            ;
        }
        catch (err) {
            var e = new SyntaxError(
                (err.message || err) + ' while parsing ' + file
            );
            e.type = 'syntax';
            e.filename = file;
            return this.emit('error', e);
        }
        
        var globals = {};
        
        varNames.forEach(function (name) {
            if (!/\./.test(name)) return;
            var parts = name.split('.')
            var prop = (scope.globals.implicitProperties || {})[parts[0]]
            if (!prop || prop.length !== 1 || prop[0] !== parts[1]) return;
            var value = vars[name](file, basedir);
            if (!value) return;
            globals[parts[0]] = '{'
                + JSON.stringify(parts[1]) + ':' + value + '}';
            self.emit('global', name);
        });
        varNames.forEach(function (name) {
            if (/\./.test(name)) return;
            if (globals[name]) return;
            if (scope.globals.implicit.indexOf(name) < 0) return;
            var value = vars[name](file, basedir);
            if (!value) return;
            globals[name] = value;
            self.emit('global', name);
        });
        
        this.push(closeOver(globals, source, file, opts));
        this.push(null);
    }
};

module.exports.vars = defaultVars;

function closeOver (globals, src, file, opts) {
    var keys = Object.keys(globals);
    if (keys.length === 0) return src;
    var values = keys.map(function (key) { return globals[key] });
    
    var wrappedSource;
    if (keys.length <= 3) {
        wrappedSource = '(function (' + keys.join(',') + '){\n'
            + src + '\n}).call(this,' + values.join(',') + ')'
{'fraction_non_alphanumeric': 0.16677108131813057, 'fraction_numerical': 0.002756546798646786, 'mean_word_length': 1.808585503166784, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '840123', 'n_tokens_mistral': 2341, 'n_tokens_neox': 2320, 'n_words': 518}