<commit_before>#!/usr/bin/env node

(function() {

    var fs = require('fs');
    var path = require('path');
    var resolve = require('resolve');

    var File = require('vinyl'); // The module file to be checked.

    var resolution = {}; // The dependency tree being resolved.
    var checked = []; // List of modules already checked.
    var toCheck = []; // List of modules to parse next.

    var defaultOpts = {
        parseRequires: true, // Naively parses source for require statements.
        indexCoreModules: false, //Whether to index core modules or skip them.
        indexPackageJSONDeps: true, // Indexes dependencies in pacakge.json
        indexPackageJSONOptDeps: true, // Indexes optional dependencies in pacakge.json
        indexPac: true, // Indexes dependencies in pacakge.json
        indexRequiresByName: true, // Indexes parsed dependencies required by name.
        indexRequiresByPath: false, // Indexes parsed dependencies required by path.
        recursePackageJSONDeps: true, // Recurse through package.json dependencies.
        recursePackageJSONOptDeps: true, // Recurse through package.json optional dependencies.
        recurseRequiresByName: true, //TODO: Recurse through parsed dependencies required by path.
        recurseRequiresByPath: true, // Recurse through parsed dependencies required by path.
        verbose: false, // Print logging info.
        limit: 1000000, // Maximum number of modules checked recursively. 
                        // Setting this to `1` stops the recursion at checking only the given module's dependencies.
                        // But this isn't really a recursion counter, as all dependencies beyond the first level count towards the limit individually.
        offsetPath: __dirname // All resolved modules are made relative to this path.
    };
    
    // Set opts.
    var opts = {};
    var setOpts = function(passedOpts) {
        for ( var key in defaultOpts ) {
            if ( typeof passedOpts === typeof {} && passedOpts[key] !== undefined ) {
                opts[key] = passedOpts[key];
            } else {
                opts[key] = defaultOpts[key];
            }
        }
    };

    // Actual logic for parsing files.
    var checkByFile = function(file) {

        // Check for the "recursion" limit.
        if (opts.limit >= 0) {
            opts.limit--;
        }
        else {
            throw "Recursion limit reached.";
        }

        var basedir = path.resolve(path.dirname(file.path));

        // Parse package.json.
        if ( opts.indexPackageJSONDeps || opts.indexPackageJSONOptDeps || opts.recursePackageJSONDeps || opts.recursePackageJSONOptDeps ) {
            require('closest-package').sync(basedir, function(json, filename) {
                
                if ( opts.indexPackageJSONDeps || opts.recursePackageJSONDeps ) {
                    for (var moduleName in json.dependencies) {
    
                        // Resolve module path.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);
    
                        // Add resolution to output.
                        if (opts.indexPackageJSONDeps) {
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                            if (opts.verbose) console.log("Indexed package.json dependency `" + moduleName + "`.");
                        }
    
                        // Add dependency to be checked recursively.
                        if (opts.recursePackageJSONDeps) {
                            toCheck.push(modulePathAbsolute);
                        }
    
                    }
                }
                
                if ( opts.indexPackageJSONOptDeps || opts.recursePackageJSONOptDeps ) {
                    for (var moduleName in json.optionalDependencies) {
    
                        // Resolve module path.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);
    
                        // Add resolution to output.
                        if (opts.indexPackageJSONOptDeps) {
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                            if (opts.verbose) console.log("Indexed package.json optional dependency `" + moduleName + "`.");
                        }
    
                        // Add dependency to be checked recursively.
                        if (opts.recursePackageJSONOptDeps) {
                            toCheck.push(modulePathAbsolute);
                        }
    
                    }
                }
                
                return true; //NOTE: Because this is a filtering function and the lookup would onlt stop at true. And we do want the very closest one.
            });
        }

        // Parse file body for requires.

        if (opts.parseRequires) {

            var input = file.contents.toString('utf-8');

            var regex = /\brequire\(([^\)]*)\)/g;

            var matchObject;
            while (matchObject = regex.exec(input)) {

                var fullMatch = matchObject[0];
                var match = matchObject[1];

                try {

                    // Evaluate and "clean" module name.
                    var moduleName = eval(match);

                    if (resolve.isCore(moduleName)) {
                        if (opts.indexCoreModules) {
                            // Make path absolute.
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = moduleName;
                            if (opts.verbose) console.log("Indexed core module `" + moduleName + "`.");
                        } else {
                            // Throw a fit if module is core to Node.
                            throw "Core modules do not run in the browser!";
                        }
                    }
                    else {

                        // Else continue.
                        // Check if module was required by path.
                        var requiredByPath = /^(?:\.{0,2})\//.test(moduleName);

                        // Check whether to parse path'd modules.
                        if (!opts.recurseRequiresByPath && requiredByPath) {
                            throw "Will not parse modules required by path.";
                        }

                        // Resolve module.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);

                        // Add resolution to index of module wasn't required in path form.
                        if (opts.indexRequiresByName && (opts.indexRequiresByPath || !requiredByPath)) {
                            if (path.resolve(file.path) != modulePathAbsolute) {
                                if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                                resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                                if (opts.verbose) console.log("Indexed required module `" + moduleName + "`.");
                            }
                        }
                        else {
                            if (opts.verbose) console.log("Did not index required module `" + moduleName + "`.");
                        }

                        // Add dependency to be checked recursively.
                        if (toCheck.indexOf(modulePathAbsolute) <= -1) {
                            toCheck.push(modulePathAbsolute);
                            if (opts.verbose) console.log("Will not parse path dependency " + moduleName + "`.");<commit_msg>Linted.<commit_after>#!/usr/bin/env node

(function() {

    var fs = require('fs');
    var path = require('path');
    var resolve = require('resolve');

    var File = require('vinyl'); // The module file to be checked.

    var resolution = {}; // The dependency tree being resolved.
    var checked = []; // List of modules already checked.
    var toCheck = []; // List of modules to parse next.

    var defaultOpts = {
        parseRequires: true, // Naively parses source for require statements.
        indexCoreModules: false, //Whether to index core modules or skip them.
        indexPackageJSONDeps: true, // Indexes dependencies in pacakge.json
        indexPackageJSONOptDeps: true, // Indexes optional dependencies in pacakge.json
        indexPac: true, // Indexes dependencies in pacakge.json
        indexRequiresByName: true, // Indexes parsed dependencies required by name.
        indexRequiresByPath: false, // Indexes parsed dependencies required by path.
        recursePackageJSONDeps: true, // Recurse through package.json dependencies.
        recursePackageJSONOptDeps: true, // Recurse through package.json optional dependencies.
        recurseRequiresByName: true, //TODO: Recurse through parsed dependencies required by path.
        recurseRequiresByPath: true, // Recurse through parsed dependencies required by path.
        verbose: false, // Print logging info.
        limit: 1000000, // Maximum number of modules checked recursively. 
                        // Setting this to `1` stops the recursion at checking only the given module's dependencies.
                        // But this isn't really a recursion counter, as all dependencies beyond the first level count towards the limit individually.
        offsetPath: __dirname // All resolved modules are made relative to this path.
    };
    
    // Set opts.
    var opts = {};
    var setOpts = function(passedOpts) {
        for ( var key in defaultOpts ) {
            if ( typeof passedOpts === typeof {} && passedOpts[key] !== undefined ) {
                opts[key] = passedOpts[key];
            } else {
                opts[key] = defaultOpts[key];
            }
        }
    };

    // Actual logic for parsing files.
    var checkByFile = function(file) {

        // Check for the "recursion" limit.
        if (opts.limit >= 0) {
            opts.limit--;
        }
        else {
            throw "Recursion limit reached.";
        }

        var basedir = path.resolve(path.dirname(file.path));

        // Parse package.json.
        if ( opts.indexPackageJSONDeps || opts.indexPackageJSONOptDeps || opts.recursePackageJSONDeps || opts.recursePackageJSONOptDeps ) {
            require('closest-package').sync(basedir, function(json, filename) {
                
                if ( opts.indexPackageJSONDeps || opts.recursePackageJSONDeps ) {
                    for (var moduleName in json.dependencies) {
    
                        // Resolve module path.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);
    
                        // Add resolution to output.
                        if (opts.indexPackageJSONDeps) {
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                            if (opts.verbose) console.log("Indexed package.json dependency `" + moduleName + "`.");
                        }
    
                        // Add dependency to be checked recursively.
                        if  (opts.recursePackageJSONDeps && toCheck.indexOf(modulePathAbsolute) <= -1 ) {
                            toCheck.push(modulePathAbsolute);
                        }
    
                    }
                }
                
                if ( opts.indexPackageJSONOptDeps || opts.recursePackageJSONOptDeps ) {
                    for (var moduleName in json.optionalDependencies) {
    
                        // Resolve module path.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);
    
                        // Add resolution to output.
                        if (opts.indexPackageJSONOptDeps) {
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                            if (opts.verbose) console.log("Indexed package.json optional dependency `" + moduleName + "`.");
                        }
    
                        // Add dependency to be checked recursively.
                        if ( opts.recursePackageJSONOptDeps && toCheck.indexOf(modulePathAbsolute) <= -1 ) {
                            toCheck.push(modulePathAbsolute);
                        }
    
                    }
                }
                
                return true; //NOTE: Because this is a filtering function and the lookup would onlt stop at true. And we do want the very closest one.
            });
        }

        // Parse file body for requires.

        if (opts.parseRequires) {

            var input = file.contents.toString('utf-8');

            var regex = /\brequire\(([^\)]*)\)/g;

            var matchObject;
            while (matchObject = regex.exec(input)) {

                var fullMatch = matchObject[0];
                var match = matchObject[1];

                try {

                    // Evaluate and "clean" module name.
                    var moduleName = eval(match);

                    if (resolve.isCore(moduleName)) {
                        if (opts.indexCoreModules) {
                            // Make path absolute.
                            if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                            resolution[basedir][moduleName] = moduleName;
                            if (opts.verbose) console.log("Indexed core module `" + moduleName + "`.");
                        } else {
                            // Throw a fit if module is core to Node.
                            throw "Core modules do not run in the browser!";
                        }
                    }
                    else {

                        // Else continue.
                        // Check if module was required by path.
                        var requiredByPath = /^(?:\.{0,2})\//.test(moduleName);

                        // Check whether to parse path'd modules.
                        if (!opts.recurseRequiresByPath && requiredByPath) {
                            throw "Will not parse modules required by path.";
                        }

                        // Resolve module.
                        var modulePathLocal = resolve.sync(moduleName, {
                            basedir: basedir
                        });
                        // Make path absolute.
                        var modulePathAbsolute = path.resolve(modulePathLocal);

                        // Add resolution to index of module wasn't required in path form.
                        if (opts.indexRequiresByName && (opts.indexRequiresByPath || !requiredByPath)) {
                            if (path.resolve(file.path) != modulePathAbsolute) {
                                if (typeof resolution[basedir] !== typeof {}) resolution[basedir] = {};
                                resolution[basedir][moduleName] = path.relative(opts.offsetPath, modulePathAbsolute);
                                if (opts.verbose) console.log("Indexed required module `" + moduleName + "`.");
                            }
                        }
                        else {
                            if (opts.verbose) console.log("Did not index required module `" + moduleName + "`.");
                        }

                        // Add dependency to be checked recursively.
                        if (toCheck.indexOf(modulePathAbsolute) <= -1) {
                            toCheck.push(modulePathAbsolute);
                            if (opts.verbose) console.log("Will recurse `" + moduleName + "`.");
{'fraction_non_alphanumeric': 0.0900029146021568, 'fraction_numerical': 0.0018653453803555815, 'mean_word_length': 1.3376481809510832, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '150417', 'n_tokens_mistral': 4000, 'n_tokens_neox': 3783, 'n_words': 1095}