<commit_before>'use strict'

const path = require('path')
const fs = require('fs').promises
const assert = require('assert')
const Journalist = require('journalist')
const Magazine = require('magazine')

const { coalesce } = require('extant')

const Operation = require('operation')
const Recorder = require('transcript/recorder')
const Strata = { Error: require('./error') }
const Storage = require('./storage')
const Player = require('transcript/player')

function _recordify (recorder, header, parts = []) {
    return recorder([[ Buffer.from(JSON.stringify(header)) ].concat(parts)])
}

function _path (...vargs) {
    return path.join.apply(path, vargs.map(varg => String(varg)))
}

// Sort function for file names that orders by their creation order.
const appendable = require('./appendable')

class FileSystem {
    static async open (options) {
        const { directory, handles, serializer, extractor, checksum } = Storage.options(options)
        const recorder = Recorder.create(checksum)
        function _path (...vargs) {
            vargs.unshift(directory)
            return path.resolve.apply(path, vargs.map(varg => String(varg)))
        }
        if (options.create) {
            const stat = await Strata.Error.resolve(fs.stat(directory), 'IO_ERROR')
            Strata.Error.assert(stat.isDirectory(), 'CREATE_NOT_DIRECTORY', { directory })
            const dir = await Strata.Error.resolve(fs.readdir(directory), 'IO_ERROR')
            Strata.Error.assert(dir.every(file => /^\./.test(file)), 'CREATE_NOT_EMPTY', { directory })
            await Strata.Error.resolve(fs.mkdir(_path('instances')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('pages')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('instances', '1'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('page'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance', '0.0'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance', '0.1')), 'IO_ERROR')
            const buffers = [ _recordify(recorder, { method: 'length', length: 1 }), _recordify(recorder, { method: 'insert', index: 0, id: '0.1' }, []) ]
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.0', 'page'), Buffer.concat(buffers), { flag: 'as' }), 'IO_ERROR')
            const zero = _recordify(recorder, { method: '0.0' })
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.1', '0.0'), zero, { flag: 'as' }), 'IO_ERROR')
            const one = _recordify(recorder, { method: 'load', page: '0.1', log: '0.0' })
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.1', '0.1'), one, { flag: 'as' }), 'IO_ERROR')
            const journalist = await Journalist.create(directory)
            journalist.mkdir('pages/0.0')
            journalist.mkdir('pages/0.1')
            journalist.rename('balance/0.0/page', 'pages/0.0/page')
            journalist.rename('balance/0.1/0.0', 'pages/0.1/0.0')
            journalist.rename('balance/0.1/0.1', 'pages/0.1/0.1')
            journalist.rmdir('balance/0.0')
            journalist.rmdir('balance/0.1')
            await journalist.prepare()
            await journalist.commit()
            await journalist.dispose()
            return { handles, recorder, directory, instance: 1, extractor, serializer, checksum, pageId: 2 }
        }
        // **TODO** Run commit log on reopen.
        const dir = await Strata.Error.resolve(fs.readdir(_path('instances')), 'IO_ERROR')
        const instances = dir
            .filter(file => /^\d+$/.test(file))
            .map(file => +file)
            .sort((left, right) => right - left)
        const instance = instances[0] + 1
        await Strata.Error.resolve(fs.mkdir(_path('instances', instance)), 'IO_ERROR')
        for (const instance of instances) {
            await Strata.Error.resolve(coalesce(fs.rm, fs.rmdir).call(fs, _path('instances', instance), { recursive: true }), 'IO_ERROR')
        }
        return { handles, recorder, directory, instance, extractor, serializer, checksum, pageId: 0 }
    }

    //

    // **TODO** Really need to think of some rules for failure. They may be
    // interim rules, like alpha release rules. For now we ask that you let
    // stuff crash, provide as much detail as Strata will provide, we
    // probably can't fix anything other than to try to make those crash
    // reports better for the future.

    // `libuv` suppresses `EPROGRESS` and `EINTR` errors treating them as
    // successful, so the only remaining error would be `EIO` which probably
    // means that the file is corrupt.

    // We want this class to be independent of an particular strata so it
    // can be shared.

    //
    static Reader = class {
        constructor (directory, options = {}) {
            options = Storage.options(options)
            this.directory = directory
            this.serializer = options.serializer
            this.extractor = options.extractor
            this.checksum = options.checksum
        }

        _path (...vargs) {
            vargs.unshift(this.directory)<commit_msg>Use old `rmdir`.<commit_after>'use strict'

const path = require('path')
const fs = require('fs').promises
const assert = require('assert')
const Journalist = require('journalist')
const Magazine = require('magazine')

const { coalesce } = require('extant')

const Operation = require('operation')
const Recorder = require('transcript/recorder')
const Strata = { Error: require('./error') }
const Storage = require('./storage')
const Player = require('transcript/player')

function _recordify (recorder, header, parts = []) {
    return recorder([[ Buffer.from(JSON.stringify(header)) ].concat(parts)])
}

function _path (...vargs) {
    return path.join.apply(path, vargs.map(varg => String(varg)))
}

// Sort function for file names that orders by their creation order.
const appendable = require('./appendable')

class FileSystem {
    static async open (options) {
        const { directory, handles, serializer, extractor, checksum } = Storage.options(options)
        const recorder = Recorder.create(checksum)
        function _path (...vargs) {
            vargs.unshift(directory)
            return path.resolve.apply(path, vargs.map(varg => String(varg)))
        }
        if (options.create) {
            const stat = await Strata.Error.resolve(fs.stat(directory), 'IO_ERROR')
            Strata.Error.assert(stat.isDirectory(), 'CREATE_NOT_DIRECTORY', { directory })
            const dir = await Strata.Error.resolve(fs.readdir(directory), 'IO_ERROR')
            Strata.Error.assert(dir.every(file => /^\./.test(file)), 'CREATE_NOT_EMPTY', { directory })
            await Strata.Error.resolve(fs.mkdir(_path('instances')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('pages')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance')), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('instances', '1'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('page'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance', '0.0'), { recursive: true }), 'IO_ERROR')
            await Strata.Error.resolve(fs.mkdir(_path('balance', '0.1')), 'IO_ERROR')
            const buffers = [ _recordify(recorder, { method: 'length', length: 1 }), _recordify(recorder, { method: 'insert', index: 0, id: '0.1' }, []) ]
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.0', 'page'), Buffer.concat(buffers), { flag: 'as' }), 'IO_ERROR')
            const zero = _recordify(recorder, { method: '0.0' })
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.1', '0.0'), zero, { flag: 'as' }), 'IO_ERROR')
            const one = _recordify(recorder, { method: 'load', page: '0.1', log: '0.0' })
            await Strata.Error.resolve(fs.writeFile(_path('balance', '0.1', '0.1'), one, { flag: 'as' }), 'IO_ERROR')
            const journalist = await Journalist.create(directory)
            journalist.mkdir('pages/0.0')
            journalist.mkdir('pages/0.1')
            journalist.rename('balance/0.0/page', 'pages/0.0/page')
            journalist.rename('balance/0.1/0.0', 'pages/0.1/0.0')
            journalist.rename('balance/0.1/0.1', 'pages/0.1/0.1')
            journalist.rmdir('balance/0.0')
            journalist.rmdir('balance/0.1')
            await journalist.prepare()
            await journalist.commit()
            await journalist.dispose()
            return { handles, recorder, directory, instance: 1, extractor, serializer, checksum, pageId: 2 }
        }
        // **TODO** Run commit log on reopen.
        const dir = await Strata.Error.resolve(fs.readdir(_path('instances')), 'IO_ERROR')
        const instances = dir
            .filter(file => /^\d+$/.test(file))
            .map(file => +file)
            .sort((left, right) => right - left)
        const instance = instances[0] + 1
        await Strata.Error.resolve(fs.mkdir(_path('instances', instance)), 'IO_ERROR')
        for (const instance of instances) {
            await Strata.Error.resolve(fs.rmdir(_path('instances', instance)), 'IO_ERROR')
        }
        return { handles, recorder, directory, instance, extractor, serializer, checksum, pageId: 0 }
    }

    //

    // **TODO** Really need to think of some rules for failure. They may be
    // interim rules, like alpha release rules. For now we ask that you let
    // stuff crash, provide as much detail as Strata will provide, we
    // probably can't fix anything other than to try to make those crash
    // reports better for the future.

    // `libuv` suppresses `EPROGRESS` and `EINTR` errors treating them as
    // successful, so the only remaining error would be `EIO` which probably
    // means that the file is corrupt.

    // We want this class to be independent of an particular strata so it
    // can be shared.

    //
    static Reader = class {
        constructor (directory, options = {}) {
            options = Storage.options(options)
            this.directory = directory
            this.serializer = options.serializer
            this.extractor = options.extractor
            this.checksum = options.checksum
        }

        _path (...vargs) {
            vargs.unshift(this.directory)
{'fraction_non_alphanumeric': 0.14865632096896292, 'fraction_numerical': 0.01097653292959879, 'mean_word_length': 2.7692582025677606, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2916112', 'n_tokens_mistral': 3131, 'n_tokens_neox': 3117, 'n_words': 777}