<commit_before>var util = require('util');
var mu = require('mu2');
var Convert = require('ansi-to-html');

function htmlFromText(text) {
    var tagsToReplace = {
        '&': '&amp;',
        '<': '&lt;'
    };
    return text.replace(/[&<]/g, function(tag) {
        return tagsToReplace[tag];
    });
}

module.exports = function (subprocess, res) {
    var convert = new Convert();

    res.writeHead(200, {
        "Content-Type": "text/html;charset=utf8"
    });

    var stream = mu.compileAndRender(
        'subprocess.mu.html', {
            title: subprocess.title,
            commandLine: subprocess.cmd.join(' '),
            running: subprocess.exitCode == null,
            exitCode: subprocess.exitCode,
            exitCodeClass: (subprocess.exitCode == null ? "" : (subprocess.exitCode === 0 ? "good" : "bad")),
            pid: subprocess.pid,
            lines: subprocess.output.map(function (line) {
                return {
                    class: line.stream,
                    html: convert.toHtml(htmlFromText(line.line))
                };
            })
        });

    // We apparently trigger a bug in the core stream library
    // if we simply use stream.pipe(res);
    stream.on('data', function (d) { res.write(d); });
    stream.on('end', function (d) { res.end(d); });
};
<commit_msg>Show only the last 1000 lines of output. The current approach does not scale too well<commit_after>var util = require('util');
var mu = require('mu2');
var Convert = require('ansi-to-html');

function htmlFromText(text) {
    var tagsToReplace = {
        '&': '&amp;',
        '<': '&lt;'
    };
    return text.replace(/[&<]/g, function(tag) {
        return tagsToReplace[tag];
    });
}

module.exports = function (subprocess, res) {
    var convert = new Convert();

    res.writeHead(200, {
        "Content-Type": "text/html;charset=utf8"
    });

    var stream = mu.compileAndRender(
        'subprocess.mu.html', {
            title: subprocess.title,
            commandLine: subprocess.cmd.join(' '),
            running: subprocess.exitCode == null,
            exitCode: subprocess.exitCode,
            exitCodeClass: (subprocess.exitCode == null ? "" : (subprocess.exitCode === 0 ? "good" : "bad")),
            pid: subprocess.pid,
            lines: subprocess.output.slice(-1000).map(function (line) {
                return {
                    class: line.stream,
                    html: convert.toHtml(htmlFromText(line.line))
                };
            })
        });

    // We apparently trigger a bug in the core stream library
    // if we simply use stream.pipe(res);
    stream.on('data', function (d) { res.write(d); });
    stream.on('end', function (d) { res.end(d); });
};

{'fraction_non_alphanumeric': 0.15578256110908428, 'fraction_numerical': 0.0072966070777088655, 'mean_word_length': 2.140893470790378, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3611744', 'n_tokens_mistral': 840, 'n_tokens_neox': 812, 'n_words': 183}