<commit_before>var flat = require('flat-tree')
var rle = require('bitfield-rle')
var pager = require('memory-pager')
var bitfield = require('sparse-bitfield')

var INDEX_UPDATE_MASK = [63, 207, 243, 252]
var INDEX_ITERATE_MASK = [0, 192, 240, 252]
var DATA_ITERATE_MASK = [128, 192, 224, 240, 248, 252, 254, 255]
var DATA_UPDATE_MASK = [127, 191, 223, 239, 247, 251, 253, 254]
var MAP_PARENT_RIGHT = new Array(256)
var MAP_PARENT_LEFT = new Array(256)
var NEXT_DATA_0_BIT = new Array(256)
var NEXT_INDEX_0_BIT = new Array(256)

for (var i = 0; i < 256; i++) {
  var a = (i & (15 << 4)) >> 4
  var b = i & 15
  MAP_PARENT_RIGHT[i] = ((a === 15 ? 3 : a === 0 ? 0 : 1) << 2) | (b === 15 ? 3 : b === 0 ? 0 : 1)
  MAP_PARENT_LEFT[i] = MAP_PARENT_RIGHT[i] << 4
  NEXT_DATA_0_BIT[i] = i === 255 ? -1 : (8 - Math.ceil(Math.log(256 - i) / Math.log(2)))
  NEXT_INDEX_0_BIT[i] = i === 255 ? -1 : Math.floor(NEXT_DATA_0_BIT[i] / 2)
}

module.exports = Bitfield

function Bitfield (buffer) {
  if (!(this instanceof Bitfield)) return new Bitfield(buffer)

  this.pages = pager(3328)

  if (buffer) {
    for (var i = 0; i < buffer.length; i += 3328) {
      this.pages.set(i / 3328, buffer.slice(i, i + 3328))
    }
  }

  this.data = bitfield({
    pageSize: 1024,
    pageOffset: 0,
    pages: this.pages,
    trackUpdates: true
  })

  this.tree = bitfield({
    pageSize: 2048,
    pageOffset: 1024,
    pages: this.pages,
    trackUpdates: true
  })

  this.index = bitfield({
    pageSize: 256,
    pageOffset: 1024 + 2048,
    pages: this.pages,
    trackUpdates: true
  })

  this.length = this.data.length
  this._iterator = flat.iterator(0)
}

Bitfield.prototype.set = function (i, value) {
  var o = i & 7
  i = (i - o) / 8
  var v = value ? this.data.getByte(i) | (128 >> o) : this.data.getByte(i) & DATA_UPDATE_MASK[o]

  if (!this.data.setByte(i, v)) return false
  this.length = this.data.length
  this._setIndex(i, v)
  return true
}

Bitfield.prototype.get = function (i) {
  return this.data.get(i)
}

// TODO: use the index to speed this up *a lot*
Bitfield.prototype.compress = function () {
  return rle.encode(this.data.toBuffer())
}

Bitfield.prototype._setIndex = function (i, value) {
  //                    (a + b | c + d | e + f | g + h)
  // -> (a | b | c | d)                                (e | f | g | h)
  //

  var o = i & 3
  i = (i - o) / 4

  var bitfield = this.index
  var ite = this._iterator
  var start = 2 * i
  var byte = (bitfield.getByte(start) & INDEX_UPDATE_MASK[o]) | (getIndexValue(value) >> (2 * o))
  var len = bitfield.length
  var maxLength = this.pages.length * 256

  ite.seek(start)

  while (ite.index < maxLength && bitfield.setByte(ite.index, byte)) {
    if (ite.isLeft()) {
      byte = MAP_PARENT_LEFT[byte] | MAP_PARENT_RIGHT[bitfield.getByte(ite.sibling())]
    } else {
      byte = MAP_PARENT_RIGHT[byte] | MAP_PARENT_LEFT[bitfield.getByte(ite.sibling())]
    }
    ite.parent()
  }

  if (len !== bitfield.length) this._expand(len)

  return ite.index !== start
}

Bitfield.prototype._expand = function (len) {
  var roots = flat.fullRoots(2 * len)
  var bitfield = this.index
  var ite = this._iterator
  var byte = 0

  for (var i = 0; i < roots.length; i++) {
    ite.seek(roots[i])
    byte = bitfield.getByte(ite.index)

    do {
      if (ite.isLeft()) {
        byte = MAP_PARENT_LEFT[byte] | MAP_PARENT_RIGHT[bitfield.getByte(ite.sibling())]
      } else {
        byte = MAP_PARENT_RIGHT[byte] | MAP_PARENT_LEFT[bitfield.getByte(ite.sibling())]
      }
    } while (setByteNoAlloc(bitfield, ite.parent(), byte))
  }
}

function setByteNoAlloc (bitfield, i, b) {
  if (i >= 8 * bitfield.length) return false
  return bitfield.setByte(i, b)
}

Bitfield.prototype.iterator = function (start, end) {
  var ite = new Iterator(this)<commit_msg>multiplied the wrong side of the if<commit_after>var flat = require('flat-tree')
var rle = require('bitfield-rle')
var pager = require('memory-pager')
var bitfield = require('sparse-bitfield')

var INDEX_UPDATE_MASK = [63, 207, 243, 252]
var INDEX_ITERATE_MASK = [0, 192, 240, 252]
var DATA_ITERATE_MASK = [128, 192, 224, 240, 248, 252, 254, 255]
var DATA_UPDATE_MASK = [127, 191, 223, 239, 247, 251, 253, 254]
var MAP_PARENT_RIGHT = new Array(256)
var MAP_PARENT_LEFT = new Array(256)
var NEXT_DATA_0_BIT = new Array(256)
var NEXT_INDEX_0_BIT = new Array(256)

for (var i = 0; i < 256; i++) {
  var a = (i & (15 << 4)) >> 4
  var b = i & 15
  MAP_PARENT_RIGHT[i] = ((a === 15 ? 3 : a === 0 ? 0 : 1) << 2) | (b === 15 ? 3 : b === 0 ? 0 : 1)
  MAP_PARENT_LEFT[i] = MAP_PARENT_RIGHT[i] << 4
  NEXT_DATA_0_BIT[i] = i === 255 ? -1 : (8 - Math.ceil(Math.log(256 - i) / Math.log(2)))
  NEXT_INDEX_0_BIT[i] = i === 255 ? -1 : Math.floor(NEXT_DATA_0_BIT[i] / 2)
}

module.exports = Bitfield

function Bitfield (buffer) {
  if (!(this instanceof Bitfield)) return new Bitfield(buffer)

  this.pages = pager(3328)

  if (buffer) {
    for (var i = 0; i < buffer.length; i += 3328) {
      this.pages.set(i / 3328, buffer.slice(i, i + 3328))
    }
  }

  this.data = bitfield({
    pageSize: 1024,
    pageOffset: 0,
    pages: this.pages,
    trackUpdates: true
  })

  this.tree = bitfield({
    pageSize: 2048,
    pageOffset: 1024,
    pages: this.pages,
    trackUpdates: true
  })

  this.index = bitfield({
    pageSize: 256,
    pageOffset: 1024 + 2048,
    pages: this.pages,
    trackUpdates: true
  })

  this.length = this.data.length
  this._iterator = flat.iterator(0)
}

Bitfield.prototype.set = function (i, value) {
  var o = i & 7
  i = (i - o) / 8
  var v = value ? this.data.getByte(i) | (128 >> o) : this.data.getByte(i) & DATA_UPDATE_MASK[o]

  if (!this.data.setByte(i, v)) return false
  this.length = this.data.length
  this._setIndex(i, v)
  return true
}

Bitfield.prototype.get = function (i) {
  return this.data.get(i)
}

// TODO: use the index to speed this up *a lot*
Bitfield.prototype.compress = function () {
  return rle.encode(this.data.toBuffer())
}

Bitfield.prototype._setIndex = function (i, value) {
  //                    (a + b | c + d | e + f | g + h)
  // -> (a | b | c | d)                                (e | f | g | h)
  //

  var o = i & 3
  i = (i - o) / 4

  var bitfield = this.index
  var ite = this._iterator
  var start = 2 * i
  var byte = (bitfield.getByte(start) & INDEX_UPDATE_MASK[o]) | (getIndexValue(value) >> (2 * o))
  var len = bitfield.length
  var maxLength = this.pages.length * 256

  ite.seek(start)

  while (ite.index < maxLength && bitfield.setByte(ite.index, byte)) {
    if (ite.isLeft()) {
      byte = MAP_PARENT_LEFT[byte] | MAP_PARENT_RIGHT[bitfield.getByte(ite.sibling())]
    } else {
      byte = MAP_PARENT_RIGHT[byte] | MAP_PARENT_LEFT[bitfield.getByte(ite.sibling())]
    }
    ite.parent()
  }

  if (len !== bitfield.length) this._expand(len)

  return ite.index !== start
}

Bitfield.prototype._expand = function (len) {
  var roots = flat.fullRoots(2 * len)
  var bitfield = this.index
  var ite = this._iterator
  var byte = 0

  for (var i = 0; i < roots.length; i++) {
    ite.seek(roots[i])
    byte = bitfield.getByte(ite.index)

    do {
      if (ite.isLeft()) {
        byte = MAP_PARENT_LEFT[byte] | MAP_PARENT_RIGHT[bitfield.getByte(ite.sibling())]
      } else {
        byte = MAP_PARENT_RIGHT[byte] | MAP_PARENT_LEFT[bitfield.getByte(ite.sibling())]
      }
    } while (setByteNoAlloc(bitfield, ite.parent(), byte))
  }
}

function setByteNoAlloc (bitfield, i, b) {
  if (8 * i >= bitfield.length) return false
  return bitfield.setByte(i, b)
}

Bitfield.prototype.iterator = function (start, end) {
  var ite = new Iterator(this)
{'fraction_non_alphanumeric': 0.15499606608969316, 'fraction_numerical': 0.047731445056386046, 'mean_word_length': 2.9456802897051215, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58024', 'n_tokens_mistral': 3299, 'n_tokens_neox': 2905, 'n_words': 727}