<commit_before>var inherits = require('inherits')
var events = require('events')
var thunky = require('thunky')
var bulk = require('bulk-write-stream')
var from = require('from2')
var rabin = process.browser ? require('through2') : require('rabin')
var pump = require('pump')
var pumpify = require('pumpify')
var collect = require('stream-collector')
var storage = require('./storage')
var cursor = require('./cursor')
var encoding = require('hyperdrive-encoding')

module.exports = Archive

function Archive (drive, key, opts) {
  events.EventEmitter.call(this)
  var self = this

  this.options = opts || {}
  this.drive = drive
  this.live = this.options.live = !key && (this.options.live !== false)
  this.metadata = drive.core.createFeed(key, this.options)
  this.content = null
  this.key = key || this.metadata.key
  this.discoveryKey = this.metadata.discoveryKey
  this.owner = !key
  this.open = thunky(open)
  this.id = drive.id

  this._sparse = !!this.options.sparse
  this._closed = false
  this._appending = []
  this._indexBlock = -1
  this._finalized = false

  function open (cb) {
    self._open(cb)
  }
}

inherits(Archive, events.EventEmitter)

Archive.prototype.replicate = function (opts) {
  if (!opts) opts = {}
  if (isStream(opts)) opts = {stream: opts}

  assertReplication(this)

  var stream = opts.stream
  var self = this
  if (!stream) stream = opts.stream = this.metadata.replicate(opts)
  else this.metadata.replicate(opts)

  this.open(function (err) {
    if (err) return stream.destroy(err)
    if (self.content && self.content.key) self.content.replicate(opts)
  })

  return stream
}

Archive.prototype.list = function (opts, cb) {
  if (typeof opts === 'function') return this.list(null, opts)
  if (!opts) opts = {}

  var self = this
  var opened = false
  var offset = opts.offset || 0
  var live = opts.live === false ? false : (opts.live || !cb)

  return collect(from.obj(read), cb)

  function read (size, cb) {
    if (!opened) return open(size, cb)
    if (offset === self._indexBlock) offset++
    if (offset === self.metadata.blocks && !live) return cb(null, null)<commit_msg>support limit in list<commit_after>var inherits = require('inherits')
var events = require('events')
var thunky = require('thunky')
var bulk = require('bulk-write-stream')
var from = require('from2')
var rabin = process.browser ? require('through2') : require('rabin')
var pump = require('pump')
var pumpify = require('pumpify')
var collect = require('stream-collector')
var storage = require('./storage')
var cursor = require('./cursor')
var encoding = require('hyperdrive-encoding')

module.exports = Archive

function Archive (drive, key, opts) {
  events.EventEmitter.call(this)
  var self = this

  this.options = opts || {}
  this.drive = drive
  this.live = this.options.live = !key && (this.options.live !== false)
  this.metadata = drive.core.createFeed(key, this.options)
  this.content = null
  this.key = key || this.metadata.key
  this.discoveryKey = this.metadata.discoveryKey
  this.owner = !key
  this.open = thunky(open)
  this.id = drive.id

  this._sparse = !!this.options.sparse
  this._closed = false
  this._appending = []
  this._indexBlock = -1
  this._finalized = false

  function open (cb) {
    self._open(cb)
  }
}

inherits(Archive, events.EventEmitter)

Archive.prototype.replicate = function (opts) {
  if (!opts) opts = {}
  if (isStream(opts)) opts = {stream: opts}

  assertReplication(this)

  var stream = opts.stream
  var self = this
  if (!stream) stream = opts.stream = this.metadata.replicate(opts)
  else this.metadata.replicate(opts)

  this.open(function (err) {
    if (err) return stream.destroy(err)
    if (self.content && self.content.key) self.content.replicate(opts)
  })

  return stream
}

Archive.prototype.list = function (opts, cb) {
  if (typeof opts === 'function') return this.list(null, opts)
  if (!opts) opts = {}

  var self = this
  var opened = false
  var offset = opts.offset || 0
  var limit = opts.limit || Infinity
  var live = opts.live === false ? false : (opts.live || !cb)

  return collect(from.obj(read), cb)

  function read (size, cb) {
    if (!opened) return open(size, cb)
    if (offset === self._indexBlock) offset++
    if (offset === self.metadata.blocks && !live) return cb(null, null)
    if (limit--) return cb(null, null)
{'fraction_non_alphanumeric': 0.14776394651913324, 'fraction_numerical': 0.0018441678192715537, 'mean_word_length': 3.8643497757847536, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4168039', 'n_tokens_mistral': 1426, 'n_tokens_neox': 1391, 'n_words': 371}