<commit_before>var cluster = require('cluster'),
    library = require('../lib/functions.js');

var args = require('minimist')(process.argv.slice(2));

if(args.v || args.version) {
    var npmPackage = require('../package.json');
    console.log(npmPackage.name + ' ' + npmPackage.version);
    process.stdout.write('Written by ');
    for(var i in npmPackage.contributors) {
        if(npmPackage.contributors.hasOwnProperty(i)) {
            process.stdout.write(npmPackage.contributors[i].name + (i < npmPackage.contributors.length - 1 ? ',' : '\n'));
        }
    }
    process.exit();
}
if(args.h || args.help) {
    var npmPackage = require('../package.json');
    console.log('Usage: ' + npmPackage.name + ' [options]');
    console.log('\nOptions:\n' +
        '  -c, --config [config file]   Uses the configuration file specified.\n' +
        '  --no-default                 Disable loading of the default configuration\n' +
        '  -v, --version                Print version information and exit\n' +
        '  -h, --help                   Prints this help information\n' +
        '  --instances [number]         Number of childs to create. Default is CPU count\n' +
        '  -l, --cluster                Use worker processes; implied by --instances > 1');
    process.exit();
}

var instances = args.instances;
if(isNaN(instances)) {
    instances = undefined;
}
var clustered = args.l || args.cluster || instances > 1;

if(cluster.isMaster && clustered) {
    var numberOfWorkers = instances || require('os').cpus().length;
    var workersOnline = 0;
    var stopFork = false;
    
    cluster.on('online', function() {
        if(++workersOnline === numberOfWorkers) {
            library.makeInfo('All workers are now online');
        }
    });
    
    cluster.on('exit', function(worker, code) {
        if(code > 0) { // Restart the worker on an error
            library.makeWarning('Worker ' + worker.process.pid + ' has exited unexpectedly, restarting');
            cluster.fork();
        }
    });
    
    for (var i = 0; i < numberOfWorkers && !stopFork; i++) {
        var env = {};
        env[library.outputEnabledEnvironmentVariable] = i === 0;
        cluster.fork(env);
    }
} 
else {
    var nodelite = require('../lib/index.js');
    
    function resolveConfigAlias(config, entry) {
        if(typeof entry === 'string') {
            entry = resolveConfigAlias(config, config[entry]);
        }
        return entry;
    }
    
    function loadConfig(path) {
        var config = require(path);
        
        for(var property in config.hosts) {
            if(config.hosts.hasOwnProperty(property)) {
                config.hosts[property] = resolveConfigAlias(config.hosts, config.hosts[property]);
            }
        }
        
        return config;
    }
    
    // Try to load the default configuration file. If we're not able to for some reason, continue on with a warning.
    var defaultConfiguration;
    if(args.default !== false) {
        try {
            defaultConfiguration = loadConfig('../conf/config.json');
        }
        catch(e) {
            library.makeWarning('Continuing without loading a default configuration');
            defaultConfiguration = { };
        }
    }
    
    var externalConfigurations = library.buildFlatArray([ args.c, args.config ]);
    var config;<commit_msg>Made some functions in bin/index.js global<commit_after>var cluster = require('cluster'),
    library = require('../lib/functions.js');

var args = require('minimist')(process.argv.slice(2));

if(args.v || args.version) {
    var npmPackage = require('../package.json');
    console.log(npmPackage.name + ' ' + npmPackage.version);
    process.stdout.write('Written by ');
    for(var i in npmPackage.contributors) {
        if(npmPackage.contributors.hasOwnProperty(i)) {
            process.stdout.write(npmPackage.contributors[i].name + (i < npmPackage.contributors.length - 1 ? ',' : '\n'));
        }
    }
    process.exit();
}
if(args.h || args.help) {
    var npmPackage = require('../package.json');
    console.log('Usage: ' + npmPackage.name + ' [options]');
    console.log('\nOptions:\n' +
        '  -c, --config [config file]   Uses the configuration file specified.\n' +
        '  --no-default                 Disable loading of the default configuration\n' +
        '  -v, --version                Print version information and exit\n' +
        '  -h, --help                   Prints this help information\n' +
        '  --instances [number]         Number of childs to create. Default is CPU count\n' +
        '  -l, --cluster                Use worker processes; implied by --instances > 1');
    process.exit();
}

function resolveConfigAlias(config, entry) {
    if(typeof entry === 'string') {
        entry = resolveConfigAlias(config, config[entry]);
    }
    return entry;
}

function loadConfig(path) {
    var config = require(path);
    
    for(var property in config.hosts) {
        if(config.hosts.hasOwnProperty(property)) {
            config.hosts[property] = resolveConfigAlias(config.hosts, config.hosts[property]);
        }
    }
    
    return config;
}

var instances = args.instances;
if(isNaN(instances)) {
    instances = undefined;
}
var clustered = args.l || args.cluster || instances > 1;

if(cluster.isMaster && clustered) {
    var numberOfWorkers = instances || require('os').cpus().length;
    var workersOnline = 0;
    var stopFork = false;
    
    cluster.on('online', function() {
        if(++workersOnline === numberOfWorkers) {
            library.makeInfo('All workers are now online');
        }
    });
    
    cluster.on('exit', function(worker, code) {
        if(code > 0) { // Restart the worker on an error
            library.makeWarning('Worker ' + worker.process.pid + ' has exited unexpectedly, restarting');
            cluster.fork();
        }
    });
    
    for (var i = 0; i < numberOfWorkers && !stopFork; i++) {
        var env = {};
        env[library.outputEnabledEnvironmentVariable] = i === 0;
        cluster.fork(env);
    }
}
else {
    var nodelite = require('../lib/index.js');
    
    // Try to load the default configuration file. If we're not able to for some reason, continue on with a warning.
    var defaultConfiguration;
    if(args.default !== false) {
        try {
            defaultConfiguration = loadConfig('../conf/config.json');
        }
        catch(e) {
            library.makeWarning('Continuing without loading a default configuration');
            defaultConfiguration = { };
        }
    }
    
    var externalConfigurations = library.buildFlatArray([ args.c, args.config ]);
    var config;
{'fraction_non_alphanumeric': 0.13849835477116362, 'fraction_numerical': 0.0023930601256356566, 'mean_word_length': 2.2763351298383148, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2298588', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1887, 'n_words': 452}