<commit_before>var _ = require('lodash');
var child_process = require('child_process');
var request = require('request');
var path = require('path');
var Promise = require('es6-promise').Promise;
var stream = require('stream');

var db = require('../db');
var settings = require('../settings');
var settings_local = require('../settings_local');
var utils = require('../lib/utils');

const PERSONA_PORT = '9009';
const PERSONA_ENDPOINT = 'http://localhost:' + PERSONA_PORT;
const PERSONA_PATH = 'node_modules/persona-faker/app.js';

const API_PORT = '5009';
const API_ENDPOINT = 'http://localhost:' + API_PORT;

const USER_COUNT = 100;
const FAKE_GAMES = [
    {
        name: '<NAME>',
        app_url: 'http://mario.broskis'
    }, 
    {
        name: 'Halo 718',
        app_url: 'http://halo.com'
    },
    {
        name: 'Left 5 Dead',
        app_url: 'http://dead.left'
    }
];

var client = db.redis();
client.on('ready', function() {
    if (settings_local.FLUSH_DB_ON_PREFILL) {
        // FIXME: this doesn't work when the script is called
        // from outside the root directory for some reason
        console.log('flushing db...');
        client.flushdb(run);
    } else {
        run();
    }
});

function startServers() {
    function startServer(serverPath, opts) {
        var proc = child_process.spawn('node', [serverPath], {
            cwd: path.dirname(__dirname),
            env: _.extend({
                DB_PREFILL: true,
                PATH: process.env.PATH,
                PORT: opts.port
            }, opts.env || {}),
            stdio: ['ignore', 'pipe', 'pipe']
        }).on('error', function(err) {
            console.error('error running ' + opts.name + ':', err);
        }).on('close', function(code, signal) {
            console.log(opts.name, 'process closed');
        });

        // TODO: Should we even bother outputting stdout?
        ['stdout', 'stderr'].forEach(function(sname) { 
            var std_stream = new stream.Writable({decodeStrings: false});
            std_stream._write = function(chunk, encoding, callback) {
                process[sname].write(opts.name + ': ' + chunk);
                callback();
            };
            proc[sname].pipe(std_stream);
        });

        console.log('running', opts.name, '(pid: ' + proc.pid + ') on port', opts.port);
        return proc;
    }

    var persona_faker = startServer(PERSONA_PATH, {
        name: 'persona-faker', 
        port: PERSONA_PORT
    });
    var api_server = startServer('app.js', {
        name: 'galaxy-api',
        port: API_PORT,
        env: {
            PERSONA_VERIFICATION_URL: PERSONA_ENDPOINT + '/verify'
        }
    });
    
    return [persona_faker, api_server];
}

function run() {
    (function listenForServerStart() {
        const prefill_namespace = 'galaxy-db-prefill';
        const signal_names = ['api', 'persona-faker'];

        var remaining_signals = _.object(signal_names.map(function(x) { return [x, true]; } ));
        client.on('pmessage', function(pattern, channel, message) {
            var channel_info = channel.split(':');
            var namespace = channel_info[0];
            if (namespace !== prefill_namespace) {
                return;
            }

            var source = channel_info[1];
            if (remaining_signals[source]) {
                console.log(source, 'is ready');
                delete remaining_signals[source];
                if (!Object.keys(remaining_signals).length) {
                    console.log('starting prefill...');
                    startRequests();
                }
            }
        }).psubscribe(prefill_namespace + ':*');
        console.log('waiting for servers to finish launching...');
    })();

    var child_procs = startServers();

    process.on('exit', function(){
        console.log('killing child servers...');
        child_procs.forEach(function(child) {
            // Need to use SIGINT because SIGTERM won't actually terminate a node process
            // so long as its HTTP server is still running
            child.kill('SIGINT');
        });
    });

    function startRequests() {
        utils.promiseMap({
            users: createUsers(), 
            games: createGames()
        }).then(function(result) {
            var gameSlugs = result.games.map(function(json) { return json.slug; });
            var userSSAs = result.users.map(function(user) { return user.token; });

            var purchasePromise = purchaseGames(userSSAs, gameSlugs);
            var friendsPromise = createFriends(result.users);

            return utils.promiseMap({
                friends: friendsPromise,
                purchases: purchasePromise
            });
        }).then(function(result) {
            // TODO: log some form of useful output<commit_msg>Setup exit handler before calling startServers so that if we crash for some reason after creating one server but before finishing the other, we can still kill the server that we had already started<commit_after>var _ = require('lodash');
var child_process = require('child_process');
var request = require('request');
var path = require('path');
var Promise = require('es6-promise').Promise;
var stream = require('stream');

var db = require('../db');
var settings = require('../settings');
var settings_local = require('../settings_local');
var utils = require('../lib/utils');

const PERSONA_PORT = '9009';
const PERSONA_ENDPOINT = 'http://localhost:' + PERSONA_PORT;
const PERSONA_PATH = 'node_modules/persona-faker/app.js';

const API_PORT = '5009';
const API_ENDPOINT = 'http://localhost:' + API_PORT;

const USER_COUNT = 100;
const FAKE_GAMES = [
    {
        name: '<NAME>',
        app_url: 'http://mario.broskis'
    }, 
    {
        name: 'Halo 718',
        app_url: 'http://halo.com'
    },
    {
        name: 'Left 5 Dead',
        app_url: 'http://dead.left'
    }
];

var client = db.redis();
client.on('ready', function() {
    if (settings_local.FLUSH_DB_ON_PREFILL) {
        // FIXME: this doesn't work when the script is called
        // from outside the root directory for some reason
        console.log('flushing db...');
        client.flushdb(run);
    } else {
        run();
    }
});

function startServers(callback) {
    function startServer(serverPath, opts) {
        var proc = child_process.spawn('node', [serverPath], {
            cwd: path.dirname(__dirname),
            env: _.extend({
                DB_PREFILL: true,
                PATH: process.env.PATH,
                PORT: opts.port
            }, opts.env || {}),
            stdio: ['ignore', 'pipe', 'pipe']
        }).on('error', function(err) {
            console.error('error running ' + opts.name + ':', err);
        }).on('close', function(code, signal) {
            console.log(opts.name, 'process closed');
        });

        // TODO: Should we even bother outputting stdout?
        ['stdout', 'stderr'].forEach(function(sname) { 
            var std_stream = new stream.Writable({decodeStrings: false});
            std_stream._write = function(chunk, encoding, callback) {
                process[sname].write(opts.name + ': ' + chunk);
                callback();
            };
            proc[sname].pipe(std_stream);
        });

        console.log('running', opts.name, '(pid: ' + proc.pid + ') on port', opts.port);
        return proc;
    }

    callback(startServer(PERSONA_PATH, {
        name: 'persona-faker', 
        port: PERSONA_PORT
    }));
    callback(startServer('app.js', {
        name: 'galaxy-api',
        port: API_PORT,
        env: {
            PERSONA_VERIFICATION_URL: PERSONA_ENDPOINT + '/verify'
        }
    }));
}

function run() {
    (function listenForServerStart() {
        const prefill_namespace = 'galaxy-db-prefill';
        const signal_names = ['api', 'persona-faker'];

        var remaining_signals = _.object(signal_names.map(function(x) { return [x, true]; } ));
        client.on('pmessage', function(pattern, channel, message) {
            var channel_info = channel.split(':');
            var namespace = channel_info[0];
            if (namespace !== prefill_namespace) {
                return;
            }

            var source = channel_info[1];
            if (remaining_signals[source]) {
                console.log(source, 'is ready');
                delete remaining_signals[source];
                if (!Object.keys(remaining_signals).length) {
                    console.log('starting prefill...');
                    startRequests();
                }
            }
        }).psubscribe(prefill_namespace + ':*');
        console.log('waiting for servers to finish launching...');
    })();

    var child_procs = [];
    process.on('exit', function(){
        console.log('killing child servers...');
        child_procs.forEach(function(child) {
            // Need to use SIGINT because SIGTERM won't actually terminate a node process
            // so long as its HTTP server is still running
            child.kill('SIGINT');
        });
    });

    // Use a callback so that if we crash for some reason after setting up one server but
    // before finishing the next, we can still kill the processes that we had started
    startServers(function(proc) {
        child_procs.push(proc);
    });

    function startRequests() {
        utils.promiseMap({
            users: createUsers(), 
            games: createGames()
        }).then(function(result) {
            var gameSlugs = result.games.map(function(json) { return json.slug; });
            var userSSAs = result.users.map(function(user) { return user.token; });

            var purchasePromise = purchaseGames(userSSAs, gameSlugs);
            var friendsPromise = createFriends(result.users);

            return utils.promiseMap({
                friends: friendsPromise,
                purchases: purchasePromise
            });
        }).then(function(result) {
            // TODO: log some form of useful output
{'fraction_non_alphanumeric': 0.13716283716283717, 'fraction_numerical': 0.0035964035964035964, 'mean_word_length': 2.1481132075471696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1319036', 'n_tokens_mistral': 2892, 'n_tokens_neox': 2813, 'n_words': 669}