<commit_before>var protocol = require('./protocol')
var eos = require('end-of-stream')
var bitfield = require('bitfield')
var events = require('events')

var MAX_FIELD = 5 * 1024 * 1024

module.exports = function () {
  var that = new events.EventEmitter()
  var blocks = []
  var haves = bitfield(1024, {grow: MAX_FIELD})

  var peers = []
  var pending = []

  function update () {
    if (!pending.length) return

    var updated = []

    for (var i = 0; i < pending.length; i++) {
      var args = pending[i]
      var p = selectPeer(args[0])
      if (p === -1) continue
      updated.push(i)
    }

    for (var j = updated.length - 1; j >= 0; j--) {
      var args = pending.splice(updated[j], 1)[0]
      request(peers[selectPeer(args[0])], args[0], args[1])
    }
  }

  function have (index) {
    haves.set(index)
    for (var i = 0; i < peers.length; i++) {<commit_msg>added todo<commit_after>var protocol = require('./protocol')
var eos = require('end-of-stream')
var bitfield = require('bitfield')
var events = require('events')

var MAX_FIELD = 5 * 1024 * 1024

// TODO: add the hashes and flat-tree
// and store them in a level + store
// the actual data in a file/blob-store
module.exports = function () {
  var that = new events.EventEmitter()
  var blocks = []
  var haves = bitfield(1024, {grow: MAX_FIELD})

  var peers = []
  var pending = []

  function update () {
    if (!pending.length) return

    var updated = []

    for (var i = 0; i < pending.length; i++) {
      var args = pending[i]
      var p = selectPeer(args[0])
      if (p === -1) continue
      updated.push(i)
    }

    for (var j = updated.length - 1; j >= 0; j--) {
      var args = pending.splice(updated[j], 1)[0]
      request(peers[selectPeer(args[0])], args[0], args[1])
    }
  }

  function have (index) {
    haves.set(index)
    for (var i = 0; i < peers.length; i++) {
{'fraction_non_alphanumeric': 0.1615138592750533, 'fraction_numerical': 0.0255863539445629, 'mean_word_length': 2.754, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2506553', 'n_tokens_mistral': 676, 'n_tokens_neox': 633, 'n_words': 190}