<commit_before>import fs = require("fs")
import vm = require("vm")
import util = require("util")

const main_re = /((?:\/\/.*\n*)*)([^]*)/
const examples_re = /((?:\/\/.*\n*)*)([^]+?)(?=\/\/|$)/g

// README introduction
console.log(
    `#js-gems
Because it's awesome...`)

// TODO ToC

// foreach gem
for (let file of fs.readdirSync("gems")) {
    let name = file.split(".")[0]
    let title = name[0].toUpperCase() + name.slice(1).replace(/-/g, " ")
    let content = fs.readFileSync(`gems/${file}`).toString()
    let [main, examples] = content.split(/\/\/\s+Example/)

    // debug([main, examples])

    // gem title    
    console.log(`\n## ${title}\n`)

    let sandbox = vm.createContext()
    let i = 0

    let [_, text, code] = main_re.exec(main)
    console.log(text.trim())
    console.log("\n\t" + code.trim().replace(/\n/g, "\n\t"))
    run(code, sandbox)

    let m    
    while (m = examples_re.exec(examples)) {
        let [_, text, code] = m
        console.log(text.trim())
        console.log("\n\t" + code.trim().replace(/\n/g, "\n\t"))
        run(code, sandbox)
    }    
}

function run(script, sandbox) {
    console.log(vm.runInContext(script, sandbox))
}
<commit_msg>update README generator<commit_after>import fs = require("fs")
import vm = require("vm")
import util = require("util")

const re = /((?:\/\/.*\n*)*)([^]+?)(?=\/\/|$)/g

// README introduction
console.log(
    `#js-gems
Because it's awesome...`)

// TODO ToC

// foreach gem
for (let file of fs.readdirSync("gems")) {
    let name = file.split(".")[0] // TODO remove only after last dot
    let title = name[0].toUpperCase() + name.slice(1).replace(/-/g, " ")
    let content = fs.readFileSync(`gems/${file}`).toString()

    // gem title
    console.log(`\n## ${title}\n`)

    // prepare gem execution context and capture stdout
    let sandbox = vm.createContext({ console: {} })
    vm.runInContext(`
        console.log = (...obj) => { stdout += obj.join(" ") + "\\n" }`, sandbox)

    // run the main code followed by the examples    
    let m, i = 0
    while (m = re.exec(content)) {
        let [_, text, code] = m

        // intro text or explanation
        console.log(text.trim().replace(/^\/\/\s*/gm, "\n"))
        // show code
        console.log("```js\n" + code.trim() + "\n```")

        // run the code then display stdout if any and the object returned if any
        sandbox.stdout = ""
        let result = vm.runInContext(code, sandbox)
        if (sandbox.stdout)
            console.log("```\n" + sandbox.stdout + "\n```")
        if (i++ && result)
            console.log("```js\n" + util.inspect(result) + "\n```")
    }
}

{'fraction_non_alphanumeric': 0.21765371557902677, 'fraction_numerical': 0.003017729158807997, 'mean_word_length': 2.4263565891472867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3764606', 'n_tokens_mistral': 940, 'n_tokens_neox': 908, 'n_words': 215}