<commit_before>const fs = require('fs').promises
const Strata = { Error: require('./error') }

// https://stackoverflow.com/a/41976600
// https://stackoverflow.com/questions/694188/when-does-the-write-system-call-write-all-of-the-requested-buffer-versus-just
// https://stackoverflow.com/questions/31737793/posix-partial-write?rq=1
// https://stackoverflow.com/questions/694188/when-does-the-write-system-call-write-all-of-the-requested-buffer-versus-just#comment70956787_694239

exports.append = async function (handle, flush, generator, sandwich = null) {
    let buffered = 0, size = 0
    const buffers = []
    if (sandwich != null) {
        const buffer = sandwich('header')
        buffered += buffer.length
        size += buffer.length
        buffers.push(buffer)
    }
    let index = 0
    for (;;) {
        const buffer = generator(index++)
        if (buffer == null) {
            break
        }
        buffers.push(buffer)
        buffered += buffer.length
        size += buffer.length
        if (buffered >= flush) {
            await Strata.Error.resolve(handle.writev(buffers), 'IO_ERROR')
            buffers.length = 0
            buffered = 0
        }
    }
    if (sandwich != null) {
        const buffer = sandwich('footer')
        buffered += buffer.length
        size += buffer.length
        buffers.push(buffer)
    }
    if (buffers.length) {
        await Strata.Error.resolve(handle.writev(buffers), 'IO_ERROR')
    }
    return size
}

exports.play = async function (player, filename, buffer, consumer) {
    let size = 0, index = 0
    const gathered = []
    const handle = await Strata.Error.resolve(fs.open(filename), 'IO_ERROR')
    try {
        for (;;) {
            const { bytesRead } = await Strata.Error.resolve(handle.read({ buffer }), 'IO_ERROR')
            if (bytesRead == 0) {
                return { gathered, length: index, size }
            }
            size += bytesRead
            const slice = bytesRead < buffer.length ? buffer.slice(0, bytesRead) : buffer
            for (const entry of player.split(slice)) {
                const result = consumer(entry, index++)
                if (result != null) {
                    gathered.push(result)
                }
            }
        }
    } finally {
        await Strata.Error.resolve(handle.close(), 'IO_ERROR')
    }
}

exports.player = async function* (player, filename, buffer) {
    const handle = await Strata.Error.resolve(fs.open(filename), 'IO_ERROR')
    try {
        for (;;) {
            const { bytesRead } = await Strata.Error.resolve(handle.read({ buffer }), 'IO_ERROR')
            if (bytesRead == 0) {<commit_msg>Use Node.js 12 compatible call to `handle.read()`.<commit_after>const fs = require('fs').promises
const Strata = { Error: require('./error') }

// https://stackoverflow.com/a/41976600
// https://stackoverflow.com/questions/694188/when-does-the-write-system-call-write-all-of-the-requested-buffer-versus-just
// https://stackoverflow.com/questions/31737793/posix-partial-write?rq=1
// https://stackoverflow.com/questions/694188/when-does-the-write-system-call-write-all-of-the-requested-buffer-versus-just#comment70956787_694239

exports.append = async function (handle, flush, generator, sandwich = null) {
    let buffered = 0, size = 0
    const buffers = []
    if (sandwich != null) {
        const buffer = sandwich('header')
        buffered += buffer.length
        size += buffer.length
        buffers.push(buffer)
    }
    let index = 0
    for (;;) {
        const buffer = generator(index++)
        if (buffer == null) {
            break
        }
        buffers.push(buffer)
        buffered += buffer.length
        size += buffer.length
        if (buffered >= flush) {
            await Strata.Error.resolve(handle.writev(buffers), 'IO_ERROR')
            buffers.length = 0
            buffered = 0
        }
    }
    if (sandwich != null) {
        const buffer = sandwich('footer')
        buffered += buffer.length
        size += buffer.length
        buffers.push(buffer)
    }
    if (buffers.length) {
        await Strata.Error.resolve(handle.writev(buffers), 'IO_ERROR')
    }
    return size
}

exports.play = async function (player, filename, buffer, consumer) {
    let size = 0, index = 0
    const gathered = []
    const handle = await Strata.Error.resolve(fs.open(filename), 'IO_ERROR')
    try {
        for (;;) {
            const { bytesRead } = await Strata.Error.resolve(handle.read(buffer, 0, buffer.length), 'IO_ERROR')
            if (bytesRead == 0) {
                return { gathered, length: index, size }
            }
            size += bytesRead
            const slice = bytesRead < buffer.length ? buffer.slice(0, bytesRead) : buffer
            for (const entry of player.split(slice)) {
                const result = consumer(entry, index++)
                if (result != null) {
                    gathered.push(result)
                }
            }
        }
    } finally {
        await Strata.Error.resolve(handle.close(), 'IO_ERROR')
    }
}

exports.player = async function* (player, filename, buffer) {
    const handle = await Strata.Error.resolve(fs.open(filename), 'IO_ERROR')
    try {
        for (;;) {
            const { bytesRead } = await Strata.Error.resolve(handle.read(buffer, 0, buffer.length), 'IO_ERROR')
            if (bytesRead == 0) {
{'fraction_non_alphanumeric': 0.13240482822655525, 'fraction_numerical': 0.02042711234911792, 'mean_word_length': 2.1533957845433256, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6127353', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1557, 'n_words': 336}