<commit_before>#!/usr/bin/env node
"use strict";

const fs = require("fs");
const os = require("os");
const path = require("path");
const { spawn } = require("child_process");

const DEBUG = process.env.DEBUG || false;
// Maximum number of gdb processes to spawn in parallel
const MAX_PARALLEL_PROCS = +process.env.MAX_PARALLEL_PROCS || 32;

// Usage: console.log(CYAN_FMT, "This shows up in cyan!")
const CYAN_FMT = "\x1b[36m%s\x1b[0m";
const YELLOW_FMT = "\x1b[33m%s\x1b[0m";

const TEST_DIR = __dirname + "/";
const BUILD_DIR = path.join(TEST_DIR, "build");

const GDB_DEFAULT_ARGS = [
    "-batch",
    `--command=${TEST_DIR}gdb-extract-def`
];

/* Split up an array into semi-evenly sized chunks */
function chunk(source, num_chunks)
{
    const arr = source.slice();
    const ret = [];

    let rem_chunks = num_chunks;
    while(rem_chunks > 0)
    {
        // We guarantee that the entire array is processed because when rem_chunk=1 -> len/1 = len
        ret.push(arr.splice(0, Math.floor(arr.length / rem_chunks)));
        rem_chunks--;
    }
    return ret;
}
console.assert(
    JSON.stringify(chunk("0 0 1 1 2 2 2 3 3 3".split(" "), 4)) ===
        JSON.stringify([["0", "0"],
                        ["1", "1"],
                        ["2", "2", "2"],
                        ["3", "3", "3"]]),
    "Chunk"
);

const dir_files = fs.readdirSync(BUILD_DIR);
const test_files = dir_files.filter((name) => {
    return name.endsWith(".bin");
}).map(name => {
    return name.slice(0, -4);
});

const nr_of_cpus = Math.min(
    os.cpus().length || 1,
    test_files.length,
    MAX_PARALLEL_PROCS
);
console.log("[+] Using %d cpus to generate fixtures", nr_of_cpus);

const workloads = chunk(test_files, nr_of_cpus);

function test_arg_formatter(workload)
{
    return workload.map(test => {
        const test_path = path.join(BUILD_DIR, test);
        return `--eval-command=extract-state ${test_path}.bin ${test_path}.fixture`;
    });
}

function set_proc_handlers(proc, n)
{
    proc.on("close", (code) => {
        console.log(`[+] child process ${n} exited with code ${code}`);
        if(code !== 0)
        {
            process.exit(code);
        }
    });

    if(DEBUG)
    {
        proc.stdout.on("data", (data) => {
            console.log(CYAN_FMT, "stdout", `${n}: ${data}`);
        });

        proc.stderr.on("data", (data) => {
            console.log(YELLOW_FMT, "stderr", `${n}: ${data}`);
        });
    }
}

for(let i = 0; i < nr_of_cpus; i++)
{
    const gdb_args = GDB_DEFAULT_ARGS.concat(test_arg_formatter(workloads[i]));

    if(DEBUG)
    {
        console.log(CYAN_FMT, "[DEBUG]", "gdb", gdb_args.join(" "));
    }

    const gdb = spawn("gdb", gdb_args);
    set_proc_handlers(gdb, i);
}
<commit_msg>Have gen_fixtures.js use mtime to generate fixtures only when needed<commit_after>#!/usr/bin/env node
"use strict";

const fs = require("fs");
const os = require("os");
const path = require("path");
const { spawn } = require("child_process");

const DEBUG = process.env.DEBUG || false;
// Maximum number of gdb processes to spawn in parallel
const MAX_PARALLEL_PROCS = +process.env.MAX_PARALLEL_PROCS || 32;

// Usage: console.log(CYAN_FMT, "This shows up in cyan!")
const CYAN_FMT = "\x1b[36m%s\x1b[0m";
const YELLOW_FMT = "\x1b[33m%s\x1b[0m";

const TEST_DIR = __dirname + "/";
const BUILD_DIR = path.join(TEST_DIR, "build");

const GDB_DEFAULT_ARGS = [
    "-batch",
    `--command=${TEST_DIR}gdb-extract-def`
];

/* Split up an array into semi-evenly sized chunks */
function chunk(source, num_chunks)
{
    const arr = source.slice();
    const ret = [];

    let rem_chunks = num_chunks;
    while(rem_chunks > 0)
    {
        // We guarantee that the entire array is processed because when rem_chunk=1 -> len/1 = len
        ret.push(arr.splice(0, Math.floor(arr.length / rem_chunks)));
        rem_chunks--;
    }
    return ret;
}
console.assert(
    JSON.stringify(chunk("0 0 1 1 2 2 2 3 3 3".split(" "), 4)) ===
        JSON.stringify([["0", "0"],
                        ["1", "1"],
                        ["2", "2", "2"],
                        ["3", "3", "3"]]),
    "Chunk"
);

const dir_files = fs.readdirSync(BUILD_DIR);
const test_files = dir_files.filter(name => {
    return name.endsWith(".bin");
}).map(name => {
    return name.slice(0, -4);
}).filter(name => {
    const bin_file = path.join(BUILD_DIR, `${name}.bin`);
    const fixture_file = path.join(BUILD_DIR, `${name}.fixture`);
    if(!fs.existsSync(fixture_file))
    {
        return true;
    }
    return fs.statSync(bin_file).mtime > fs.statSync(fixture_file).mtime;
});

const nr_of_cpus = Math.min(
    os.cpus().length || 1,
    test_files.length,
    MAX_PARALLEL_PROCS
);
console.log("[+] Using %d cpus to generate %d fixtures", nr_of_cpus, test_files.length);

const workloads = chunk(test_files, nr_of_cpus);

function test_arg_formatter(workload)
{
    return workload.map(test => {
        const test_path = path.join(BUILD_DIR, test);
        return `--eval-command=extract-state ${test_path}.bin ${test_path}.fixture`;
    });
}

function set_proc_handlers(proc, n)
{
    proc.on("close", (code) => {
        console.log(`[+] child process ${n} exited with code ${code}`);
        if(code !== 0)
        {
            process.exit(code);
        }
    });

    if(DEBUG)
    {
        proc.stdout.on("data", (data) => {
            console.log(CYAN_FMT, "stdout", `${n}: ${data}`);
        });

        proc.stderr.on("data", (data) => {
            console.log(YELLOW_FMT, "stderr", `${n}: ${data}`);
        });
    }
}

for(let i = 0; i < nr_of_cpus; i++)
{
    const gdb_args = GDB_DEFAULT_ARGS.concat(test_arg_formatter(workloads[i]));

    if(DEBUG)
    {
        console.log(CYAN_FMT, "[DEBUG]", "gdb", gdb_args.join(" "));
    }

    const gdb = spawn("gdb", gdb_args);
    set_proc_handlers(gdb, i);
}

{'fraction_non_alphanumeric': 0.17579791773340162, 'fraction_numerical': 0.014336917562724014, 'mean_word_length': 2.8200782268578877, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2731001', 'n_tokens_mistral': 2163, 'n_tokens_neox': 2111, 'n_words': 450}