<commit_before>var ok = require('assert').ok
var path = require('path')

var cadence = require('cadence/redux')

require('cadence/loops')

var Cache = require('magazine')

var extend = require('./extend')

var Locker = require('./locker')
var Queue = require('./queue')
var Script = require('./script')

function compare (a, b) { return a < b ? -1 : a > b ? 1 : 0 }

function extract (a) { return a }

function Sheaf (options) {
    this.fs = options.fs || require('fs')
    this.nextAddress = 0
    this.directory = options.directory
    this.cache = options.cache || (new Cache)
    this.options = options
    this.tracer = options.tracer || function () { arguments[2]() }
    this.sequester = options.sequester || require('sequester')
    this.extractor = options.extractor || extract
    this.comparator = options.comparator || compare
    this.player = options.player
    this.logger = options.logger
    this.checksum = (function () {
        if (typeof options.checksum == 'function') return options.checksum
        var algorithm
        switch (algorithm = options.checksum || 'sha1') {
        case 'none':
            return function () {
                return {
                    update: function () {},
                    digest: function () { return '0' }
                }
            }
        default:
            var crypto = require('crypto')
            return function (m) { return crypto.createHash(algorithm) }
        }
    })()
    this.serialize = options.serialize || function (object) { return new Buffer(JSON.stringify(object)) }
    this.deserialize = options.deserialize || function (buffer) { return JSON.parse(buffer.toString()) }
    this.lengths = {}
}

Sheaf.prototype.create = function () {
    var root = this.createBranch({ penultimate: true })
    var leaf = this.createLeaf()
    this.splice(root, 0, 0, { address: leaf.address, heft: 0 })
    ok(root.address == 0, 'root not zero')
    return { root: root, leaf: leaf }
}

Sheaf.prototype.unbalanced = function (page, force) {
    if (force) {
        this.lengths[page.address] = this.options.leafSize
    } else if (this.lengths[page.address] == null) {
        this.lengths[page.address] = page.items.length - page.ghosts
    }
}

Sheaf.prototype.heft = function (page, s) {
    this.magazine.get(page.address).adjustHeft(s)
}

Sheaf.prototype.createLeaf = function (override) {
    return this.createPage({
        rotation: 0,
        loaders: {},
        entries: 0,
        ghosts: 0,
        items: [],
        right: { address: 0, key: null },
        queue: this.sequester.createQueue()
    }, override, 0)
}

Sheaf.prototype.createPage = function (page, override, remainder) {
    if (override.address == null) {
        while ((this.nextAddress % 2) == remainder) this.nextAddress++
        override.address = this.nextAddress++
    }
    return extend(page, override)
}

Sheaf.prototype.createBranch = function (override) {
    return this.createPage({
        items: [],
        entries: 0,
        rotation: 0,
        penultimate: true,
        queue: this.sequester.createQueue()
    }, override, 1)
}
<commit_msg>Create a `Page` object.<commit_after>var ok = require('assert').ok
var path = require('path')

var cadence = require('cadence/redux')

require('cadence/loops')

var Cache = require('magazine')

var extend = require('./extend')

var Locker = require('./locker')
var Queue = require('./queue')
var Script = require('./script')

function compare (a, b) { return a < b ? -1 : a > b ? 1 : 0 }

function extract (a) { return a }

function Sheaf (options) {
    this.fs = options.fs || require('fs')
    this.nextAddress = 0
    this.directory = options.directory
    this.cache = options.cache || (new Cache)
    this.options = options
    this.tracer = options.tracer || function () { arguments[2]() }
    this.sequester = options.sequester || require('sequester')
    this.extractor = options.extractor || extract
    this.comparator = options.comparator || compare
    this.player = options.player
    this.logger = options.logger
    this.checksum = (function () {
        if (typeof options.checksum == 'function') return options.checksum
        var algorithm
        switch (algorithm = options.checksum || 'sha1') {
        case 'none':
            return function () {
                return {
                    update: function () {},
                    digest: function () { return '0' }
                }
            }
        default:
            var crypto = require('crypto')
            return function (m) { return crypto.createHash(algorithm) }
        }
    })()
    this.serialize = options.serialize || function (object) { return new Buffer(JSON.stringify(object)) }
    this.deserialize = options.deserialize || function (buffer) { return JSON.parse(buffer.toString()) }
    this.lengths = {}
}

Sheaf.prototype.create = function () {
    var root = this.createBranch({ penultimate: true })
    var leaf = this.createLeaf()
    this.splice(root, 0, 0, { address: leaf.address, heft: 0 })
    ok(root.address == 0, 'root not zero')
    return { root: root, leaf: leaf }
}

Sheaf.prototype.unbalanced = function (page, force) {
    if (force) {
        this.lengths[page.address] = this.options.leafSize
    } else if (this.lengths[page.address] == null) {
        this.lengths[page.address] = page.items.length - page.ghosts
    }
}

Sheaf.prototype.heft = function (page, s) {
    this.magazine.get(page.address).adjustHeft(s)
}

function Page (sheaf, prototype, override, remainder) {
    if (override.address == null) {
        while (sheaf.nextAddress % 2 == remainder) sheaf.nextAddress++
        override.address = sheaf.nextAddress++
    }
    extend(this, prototype)
    extend(this, override || {})
}

Sheaf.prototype.createBranch = function (override) {
    return new Page(this, {
        items: [],
        entries: 0,
        rotation: 0,
        penultimate: true,
        queue: this.sequester.createQueue()
    }, override, 1)
}

Sheaf.prototype.createLeaf = function (override) {
    return new Page(this, {
        rotation: 0,
        loaders: {},
        entries: 0,
        ghosts: 0,
        items: [],
        right: { address: 0, key: null },
        queue: this.sequester.createQueue()
    }, override, 0)
}

{'fraction_non_alphanumeric': 0.13985345651481362, 'fraction_numerical': 0.0063714558776680474, 'mean_word_length': 2.5716723549488054, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3412446', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1852, 'n_words': 459}