<commit_before>// This software is free software. See AUTHORS and LICENSE for more
// information on the copying conditions.

"use strict";

const express = require("express");
const program = require("commander");
const scrape = require("./lib/scrape");
const test = require("./lib/test");

// Returns all roarmap institutions as JSON
let latestCheck = 0;
let cache = {};
const getInstitutions = exports.getInstitutions = (_, res) => {
    const now = new Date().getTime();
    if (now - latestCheck < 15000) {
        res.json(cache);
        return;
    }
    scrape.scrape((err, d) => {
        if (err) {
            console.log(err);
            res.status(500).send("Internal server error");
            return;
        }
        latestCheck = now;
        cache = d;
        res.json(d);
    });
};

program
    .version("0.0.4")
    .option("-d, --dump", "Scrape roarmaps database and dump it to stdout")
    .option("-t, --test", "Self test scraper and rules")
    .parse(process.argv);

if (program.dump) {
    scrape((err, data) => {
        if (err) {
            console.log(err);
            return;
        }
        console.log("Dump follows:\n\n");
        console.log(data);
    });
    return;
}

if (program.test) {
    test.test();
    return;
}

const availableApis = (_, res) => {
    res.json({
        "/api": "Return this list of APIs",
        "/api/": "Same as '/api'",
        "/api/institutions": "Scrape and return all roarmap institutions",
        "/api/version": "Returns API version",
    });
}

const app = express();
app.get("/api", availableApis);
app.get("/api/", availableApis);
app.get("/api/institutions", getInstitutions);<commit_msg>server.js: deal with unhandled errors w/ cluster (#34)<commit_after>// This software is free software. See AUTHORS and LICENSE for more
// information on the copying conditions.

"use strict";

const cluster = require("cluster");
const express = require("express");
const program = require("commander");
const scrape = require("./lib/scrape");
const test = require("./lib/test");

// Returns all roarmap institutions as JSON
let latestCheck = 0;
let cache = {};
const getInstitutions = exports.getInstitutions = (_, res) => {
    const now = new Date().getTime();
    if (now - latestCheck < 15000) {
        res.json(cache);
        return;
    }
    scrape.scrape((err, d) => {
        if (err) {
            console.log(err);
            res.status(500).send("Internal server error");
            return;
        }
        latestCheck = now;
        cache = d;
        res.json(d);
    });
};

program
    .version("0.0.4")
    .option("-d, --dump", "Scrape roarmaps database and dump it to stdout")
    .option("-t, --test", "Self test scraper and rules")
    .parse(process.argv);

if (program.dump) {
    scrape((err, data) => {
        if (err) {
            console.log(err);
            return;
        }
        console.log("Dump follows:\n\n");
        console.log(data);
    });
    return;
}

if (program.test) {
    test.test();
    return;
}

const availableApis = (_, res) => {
    res.json({
        "/api": "Return this list of APIs",
        "/api/": "Same as '/api'",
        "/api/institutions": "Scrape and return all roarmap institutions",
        "/api/version": "Returns API version",
    });
}

// Robustness model: process requests using a child process and
// respawn it in the event that it dies
if (cluster.isMaster) {
    console.log("forking worker process");
    cluster.fork();
    cluster.on("exit", (worker, code, signal) => {
        console.log("worker process died");
        setTimeout(() => {
            console.log("forking worker process");
            cluster.fork();
        }, 1000);
    });
    return;
}

const app = express();
app.get("/api", availableApis);
app.get("/api/", availableApis);
app.get("/api/institutions", getInstitutions);
{'fraction_non_alphanumeric': 0.1629591308846353, 'fraction_numerical': 0.007759958613554061, 'mean_word_length': 2.419098143236074, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4382292', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1185, 'n_words': 326}