<commit_before>const bodyParser = require('body-parser');
const cors = require('cors');
const { Router } = require('express');
const { txtFileStream } = require('../services/download.txt');
const { getIpid } = require('../services/ipid');
const xRobotsTag = require('../services/x-robots-tag');
const logger = require('../services/logger');
const cuid = require('cuid');
const { Docs } = require('../dao/dao.sqlite');
const validate = require('../services/validate');
const { generateRpCode } = require('../services/rpcode.js');
const { verifyChallenge } = require('../services/challenge');
const errors = require('../errors');
const { publish } = require('../events');

const config = require('../config');
const { generateChallenge } = require('../services/challenge');

const router = Router();
router.use(bodyParser.json());
router.use(bodyParser.urlencoded({ extended: true }));
if (config.get('cors')) router.use(cors());
router.use(xRobotsTag);

router.post('/rp.json', async (req, res, next) => {
    const rpCode = generateRpCode();
    const namespace = 'rp_' + cuid();
    const fields = await validate('meta', req.body); // TODO or throw BAD_RP
    const ipid = getIpid(req.ip);

    await Docs.create(namespace, 'meta', 'meta', fields, ipid);
    await Docs.create('system', 'urls', rpCode, { rpNamespace: namespace }, ipid);

    res.status(201).json({ rpCode });
});

router.get('/challenge.json', async (req, res, next) => {
    const challenge = await generateChallenge();
    res.status(200).json(challenge);
});

const rpGroup = '/rp/:rpCode([-0-9a-zA-Z]{1,100})';

router.get(`${rpGroup}`, async (req, res, next) => {
    // TODO transaction start
    const { rpNamespace } = await Docs.doc('system', 'urls', req.params.rpCode);

    const { title, desc } = await Docs.doc(rpNamespace, 'meta', 'meta');
    const msgs = await Docs.docs(rpNamespace, 'msgs', { reverse: true, limit: 60 }).asArray();
    msgs.reverse();
    const charas = await Docs.docs(rpNamespace, 'charas', {}).asArray();
    const lastEventId = await Docs.lastEventId();
    // TODO transaction end

    res.status(200).json({ title, desc, msgs, charas, lastEventId })
});

router.get(`${rpGroup}/updates`, async (req, res, next) => {
    res.writeHead(200, {
        'Content-Type': 'text/event-stream',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive'
    });

    res.write("data: hello\n\n");
    res.write("data: hello\n\n");
    res.write("data: hello\n\n");
});

router.get(`${rpGroup}/page/:pageNum([1-9][0-9]{0,})`, async (req, res, next) => {
    const { rpNamespace } = await Docs.doc('system', 'urls', req.params.rpCode);

    const skip = (req.params.pageNum - 1) * 20;
    const limit = 20;

    const { title, desc } = await Docs.doc(rpNamespace, 'meta', 'meta');
    const msgs = await Docs.docs(rpNamespace, 'msgs', { skip, limit }).asArray();
    const charas = await Docs.docs(rpNamespace, 'charas', {}).asArray();<commit_msg>WIP server-side events api<commit_after>const bodyParser = require('body-parser');
const cors = require('cors');
const { Router } = require('express');
const { txtFileStream } = require('../services/download.txt');
const { getIpid } = require('../services/ipid');
const xRobotsTag = require('../services/x-robots-tag');
const logger = require('../services/logger');
const cuid = require('cuid');
const { Docs } = require('../dao/dao.sqlite');
const validate = require('../services/validate');
const { generateRpCode } = require('../services/rpcode.js');
const { verifyChallenge } = require('../services/challenge');
const errors = require('../errors');
const { publish, subscribe } = require('../events');

const config = require('../config');
const { generateChallenge } = require('../services/challenge');

const router = Router();
router.use(bodyParser.json());
router.use(bodyParser.urlencoded({ extended: true }));
if (config.get('cors')) router.use(cors());
router.use(xRobotsTag);

router.post('/rp.json', async (req, res, next) => {
    const rpCode = generateRpCode();
    const namespace = 'rp_' + cuid();
    const fields = await validate('meta', req.body); // TODO or throw BAD_RP
    const ipid = getIpid(req.ip);

    await Docs.create(namespace, 'meta', 'meta', fields, ipid);
    await Docs.create('system', 'urls', rpCode, { rpNamespace: namespace }, ipid);

    res.status(201).json({ rpCode });
});

router.get('/challenge.json', async (req, res, next) => {
    const challenge = await generateChallenge();
    res.status(200).json(challenge);
});

const rpGroup = '/rp/:rpCode([-0-9a-zA-Z]{1,100})';

router.get(`${rpGroup}`, async (req, res, next) => {
    // TODO transaction start
    const { rpNamespace } = await Docs.doc('system', 'urls', req.params.rpCode);

    const { title, desc } = await Docs.doc(rpNamespace, 'meta', 'meta');
    const msgs = await Docs.docs(rpNamespace, 'msgs', { reverse: true, limit: 60 }).asArray();
    msgs.reverse();
    const charas = await Docs.docs(rpNamespace, 'charas', {}).asArray();
    const lastEventId = await Docs.lastEventId();
    // TODO transaction end

    res.status(200).json({ title, desc, msgs, charas, lastEventId })
});

router.get(`${rpGroup}/updates`, async (req, res, next) => {
    const { rpNamespace } = await Docs.doc('system', 'urls', req.params.rpCode);

    // Server-sent event headers
    res.writeHead(200, {
        'Content-Type': 'text/event-stream',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive'
    });

    // Server-sent event keep alive
    res.write(':\n\n');
    let keepAliveTimer = setInterval(() => {
        res.write(':\n\n');
    }, 10000);

    const send = (json) => {
        const { id, data } = json;
        res.write(`id: ${id}\ndata: ${JSON.stringify(data)}\n\n`);
    };
    
    // TODO get updates since ?lastMessageId=x, and send init message

    const unsub = subscribe(rpNamespace, send);

    // Stop streaming upon closing the connection
    req.once('close', () => {
        unsub();
        clearInterval(keepAliveTimer);
        res.end();
    });
});

router.get(`${rpGroup}/page/:pageNum([1-9][0-9]{0,})`, async (req, res, next) => {
    const { rpNamespace } = await Docs.doc('system', 'urls', req.params.rpCode);

    const skip = (req.params.pageNum - 1) * 20;
    const limit = 20;

    const { title, desc } = await Docs.doc(rpNamespace, 'meta', 'meta');
    const msgs = await Docs.docs(rpNamespace, 'msgs', { skip, limit }).asArray();
    const charas = await Docs.docs(rpNamespace, 'charas', {}).asArray();
{'fraction_non_alphanumeric': 0.19960171568627452, 'fraction_numerical': 0.009957107843137254, 'mean_word_length': 3.7552804078659867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3841246', 'n_tokens_mistral': 2191, 'n_tokens_neox': 2126, 'n_words': 489}