<commit_before>/*
*
* Copyright (C) 2011, The Locker Project
* All rights reserved.
*
* Please see the LICENSE file for more information.
*
*/

/* random notes:
on startup scan all folders
    Apps Collections Connectors - generate lists of "available"
    Me/* - generate lists of "existing"

when asked, run any existing and return localhost:port
if first time
    check dependencies
    create Me/ dir
    create me.json settings
    pick a port
*/

require.paths.push(__dirname + "/Common/node");
var spawn = require('child_process').spawn;
var fs = require('fs');
var path = require('path');

// This lconfig stuff has to come before and other locker modules are loaded!!
var lconfig = require('lconfig');
lconfig.load((process.argv[2] == '--config'? process.argv[3] : 'config.json'));


//var crypto = require('crypto');
var lconsole = require("lconsole");
var lscheduler = require("lscheduler");
var serviceManager = require("lservicemanager");
var dashboard = require(__dirname + "/Ops/dashboard.js");
var webservice = require(__dirname + "/Ops/webservice.js");
var lcrypto = require("lcrypto");


if(lconfig.lockerHost != "localhost" && lconfig.lockerHost != "127.0.0.1") {
    console.warn('if I\'m running on a public IP I needs to have password protection,' + // uniquely self (de?)referential? lolz!
                'which if so inclined can be hacked into lockerd.js and added since' + 
                ' it\'s apparently still not implemented :)\n\n');
}
var shuttingDown_ = false;

var mongoProcess;
path.exists(lconfig.me + '/' + lconfig.mongo.dataDir, function(exists) {
    if(!exists)
        fs.mkdirSync(lconfig.me + '/' + lconfig.mongo.dataDir, 0755);
    mongoProcess = spawn('mongod', ['--dbpath', lconfig.lockerDir + '/' + lconfig.me + '/' + lconfig.mongo.dataDir, 
                                    '--port', lconfig.mongo.port]);
    mongoProcess.stderr.on('data', function(data) {
        console.error('mongod err: ' + data);
    });
    
    var mongodExit = function(errorCode) {
        if(errorCode !== 0) {
            console.error('mongod did not start successfully.');
            shutdown();
        }
    };
    mongoProcess.on('exit', mongodExit);
    
    // watch for mongo startup
    var mongoOutput = "";
    var callback = function(data) {
        mongoOutput += data;
        if(mongoOutput.match(/ waiting for connections on port/g)) {
            mongoProcess.stdout.removeListener('data', callback);
            checkKeys();
        }
    };
    mongoProcess.stdout.on('data', callback);
});


function checkKeys() {
    lcrypto.generateSymKey(function(hasKey) {
        if (!hasKey) {
            shutdown();
            return;
        }
        lcrypto.generatePKKeys(function(hasKeys) {
            if (!hasKeys) {
                shutdown();
                return;
            }
            finishStartup();
        });
    });
}

function finishStartup() {
    // look for available things
    lconfig.scannedDirs.forEach(function(dirToScan) {
        console.log(dirToScan);
        var installable = true;
        if (dirToScan === "Collections") installable = false;
        serviceManager.scanDirectory(dirToScan, installable);
    });

    // look for existing things
    serviceManager.findInstalled();

    lscheduler.masterScheduler.loadAndStart();

    webservice.startService(lconfig.lockerPort);

    var lockerPortNext = "1"+lconfig.lockerPort;
    dashboard.start(lockerPortNext);
    lockerPortNext++;

    console.log('locker is running, use your browser and visit ' + lconfig.lockerBase);
}

function shutdown() {
    process.stdout.write("\n");
    shuttingDown_ = true;
    dashboard.instance.kill(dashboard.pid, "SIGINT");
    serviceManager.shutdown(function() {
        mongoProcess.kill();
        console.log("Shutdown complete.");
        process.exit(0);
    });
}

process.on("SIGINT", function() {
    shutdown();
});

// Export some things so this can be used by other processes, mainly for the test runner
exports.shutdown = shutdown;
<commit_msg>simple debugging help when mongod doesn't start<commit_after>/*
*
* Copyright (C) 2011, The Locker Project
* All rights reserved.
*
* Please see the LICENSE file for more information.
*
*/

/* random notes:
on startup scan all folders
    Apps Collections Connectors - generate lists of "available"
    Me/* - generate lists of "existing"

when asked, run any existing and return localhost:port
if first time
    check dependencies
    create Me/ dir
    create me.json settings
    pick a port
*/

require.paths.push(__dirname + "/Common/node");
var spawn = require('child_process').spawn;
var fs = require('fs');
var path = require('path');

// This lconfig stuff has to come before and other locker modules are loaded!!
var lconfig = require('lconfig');
lconfig.load((process.argv[2] == '--config'? process.argv[3] : 'config.json'));


//var crypto = require('crypto');
var lconsole = require("lconsole");
var lscheduler = require("lscheduler");
var serviceManager = require("lservicemanager");
var dashboard = require(__dirname + "/Ops/dashboard.js");
var webservice = require(__dirname + "/Ops/webservice.js");
var lcrypto = require("lcrypto");


if(lconfig.lockerHost != "localhost" && lconfig.lockerHost != "127.0.0.1") {
    console.warn('if I\'m running on a public IP I needs to have password protection,' + // uniquely self (de?)referential? lolz!
                'which if so inclined can be hacked into lockerd.js and added since' + 
                ' it\'s apparently still not implemented :)\n\n');
}
var shuttingDown_ = false;

var mongoProcess;
path.exists(lconfig.me + '/' + lconfig.mongo.dataDir, function(exists) {
    if(!exists)
        fs.mkdirSync(lconfig.me + '/' + lconfig.mongo.dataDir, 0755);
    mongoProcess = spawn('mongod', ['--dbpath', lconfig.lockerDir + '/' + lconfig.me + '/' + lconfig.mongo.dataDir, 
                                    '--port', lconfig.mongo.port]);
    mongoProcess.stderr.on('data', function(data) {
        console.error('mongod err: ' + data);
    });
    
    var mongoOutput = "";
    var mongodExit = function(errorCode) {
        if(errorCode !== 0) {
            console.error('mongod did not start successfully ('+errorCode+'), here was the stdout: '+mongoOutput);
            shutdown();
        }
    };
    mongoProcess.on('exit', mongodExit);
    
    // watch for mongo startup
    var callback = function(data) {
        mongoOutput += data;
        if(mongoOutput.match(/ waiting for connections on port/g)) {
            mongoProcess.stdout.removeListener('data', callback);
            checkKeys();
        }
    };
    mongoProcess.stdout.on('data', callback);
});


function checkKeys() {
    lcrypto.generateSymKey(function(hasKey) {
        if (!hasKey) {
            shutdown();
            return;
        }
        lcrypto.generatePKKeys(function(hasKeys) {
            if (!hasKeys) {
                shutdown();
                return;
            }
            finishStartup();
        });
    });
}

function finishStartup() {
    // look for available things
    lconfig.scannedDirs.forEach(function(dirToScan) {
        console.log(dirToScan);
        var installable = true;
        if (dirToScan === "Collections") installable = false;
        serviceManager.scanDirectory(dirToScan, installable);
    });

    // look for existing things
    serviceManager.findInstalled();

    lscheduler.masterScheduler.loadAndStart();

    webservice.startService(lconfig.lockerPort);

    var lockerPortNext = "1"+lconfig.lockerPort;
    dashboard.start(lockerPortNext);
    lockerPortNext++;

    console.log('locker is running, use your browser and visit ' + lconfig.lockerBase);
}

function shutdown() {
    process.stdout.write("\n");
    shuttingDown_ = true;
    dashboard.instance.kill(dashboard.pid, "SIGINT");
    serviceManager.shutdown(function() {
        mongoProcess.kill();
        console.log("Shutdown complete.");
        process.exit(0);
    });
}

process.on("SIGINT", function() {
    shutdown();
});

// Export some things so this can be used by other processes, mainly for the test runner
exports.shutdown = shutdown;

{'fraction_non_alphanumeric': 0.1340117704757234, 'fraction_numerical': 0.004659146640510054, 'mean_word_length': 2.854914933837429, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6216634', 'n_tokens_mistral': 2424, 'n_tokens_neox': 2351, 'n_words': 606}