<commit_before>/**
 * pool.js - peer management for bcoin
 * Copyright (c) 2014-2015, <NAME> (MIT License)
 * https://github.com/indutny/bcoin
 */

var async = require('async');
var inherits = require('inherits');
var EventEmitter = require('events').EventEmitter;

var bcoin = require('../bcoin');
var utils = bcoin.utils;
var assert = utils.assert;
var network = bcoin.protocol.network;

/**
 * Pool
 */

function Pool(options) {
  var self = this;
  var Chain;

  if (!(this instanceof Pool))
    return new Pool(options);

  EventEmitter.call(this);

  this.options = options || {};

  if (this.options.network)
    network.set(this.options.network);

  this.options.fullNode = !!this.options.fullNode;
  this.options.relay == null
    ? (this.options.fullNode ? false : true)
    : this.options.relay;
  this.storage = this.options.storage;
  this.destroyed = false;
  this.size = options.size || 32;
  this.parallel = options.parallel || 2000;
  this.redundancy = options.redundancy || 2;

  this.backoff = {
    delta: options.backoffDelta || 500,
    max: options.backoffMax || 5000
  };

  this.load = {
    timeout: options.loadTimeout || 3000,
    interval: options.loadInterval || 5000,
    window: options.loadWindow || 250,
    lastRange: null,
    rangeWindow: options.rangeWindow || 1000,
    timer: null,
    lwm: options.lwm || this.parallel * 2,
    hwm: options.hwm || this.parallel * 8,
    hiReached: false
  };

  if (this.options.fullNode) {
    if (!options.loadTimeout)
      this.load.timeout = 30000;
    if (!options.loadInterval)
      this.load.interval = 5000;
  }

  this.maxRetries = options.maxRetries || 42;
  this.requestTimeout = options.requestTimeout || 10000;

  Chain = this.options.fullNode
    ? bcoin.fullChain
    : bcoin.spvChain;

  this.chain = new Chain({
    storage: this.storage,
    fullNode: this.options.fullNode,
    startHeight: this.options.startHeight
  });

  this.watchMap = {};

  this.bloom = new bcoin.bloom(
    8 * 1024,
    10,
    (Math.random() * 0xffffffff) | 0
  );

  this.peers = {
    // Peers that are loading blocks themselves
    block: [],
    // Peers that are still connecting
    pending: [],
    // Peers that are loading block ids
    load: null
  };

  this.block = {
    lastHash: null,
    bestHeight: 0,
    bestHash: null
  };

  this.request = {
    map: {},
    active: 0,
    queue: []
  };

  this.validate = {
    // 5 days scan delta for obtaining TXs
    delta: 5 * 24 * 3600,

    // Minimum verification depth
    minDepth: options.minValidateDepth || 0,

    // getTX map
    map: {}
  };

  // Currently broadcasted TXs
  this.tx = {
    list: [],
    timeout: options.txTimeout || 60000
  };

  // Added and watched wallets
  this.options.wallets = this.options.wallets || [];
  this.wallets = [];

  this.createSocket = options.createConnection || options.createSocket;
  assert(this.createSocket);

  this.chain.on('debug', function() {
    var args = Array.prototype.slice.call(arguments);
    self.emit.apply(self, ['debug'].concat(args));
  });

  if (!this.chain.loading) {
    this._init();
  } else {
    this.chain.once('load', function() {
      self._init();
    });
  }
}

inherits(Pool, EventEmitter);

Pool.prototype._init = function _init() {
  var self = this;
  var i;

  this._addLoader();

  for (i = 0; i < this.size; i++)
    this._addPeer(0);

  this._load();

  this.chain.on('missing', function(hash, preload, parent) {
    if (!self.options.fullNode) {
      self._request('filtered', hash, { force: true });
      self._scheduleRequests();
      self._loadRange(preload);
    }
  });

  this.chain.on('fork', function(height, hash, checkpoint) {
    var peer = self.peers.load;

    if (!self.options.fullNode)
      return;

    if (!peer)
      return;

    peer.destroy();
  });

  this.options.wallets.forEach(function(w) {
    self.addWallet(w);
  });
};

Pool.prototype._addLoader = function _addLoader() {
  var self = this;
  var peer, interval, timer;

  if (this.destroyed)
    return;

  if (this.peers.load != null)
    return;

  peer = new bcoin.peer(this, this.createSocket, {
    backoff: 750 * Math.random(),
    startHeight: this.options.startHeight,
    relay: this.options.relay
  });

  this.peers.load = peer;

  peer.on('error', function(err) {
    self.emit('error', err, peer);
  });

  peer.once('close', onclose);
  function onclose() {
    clearTimeout(timer);
    clearInterval(interval);
    self._removePeer(peer);
    if (self.destroyed)
      return;
    self._addLoader();
  }

  peer.once('ack', function() {
    peer.updateWatch();
    if (!self._load())
      clearTimeout(timer);
  });

  peer.on('version', function(version) {
    if (version.height > self.block.bestHeight)
      self.block.bestHeight = version.height;
    self.emit('version', version, peer);
  });

  function load() {
    self._load();
  }

  interval = setInterval(load, this.load.interval);

  function destroy() {
    // Chain is full and up-to-date
    if (self.chain.isFull()) {
      clearTimeout(timer);
      self.emit('full');
      self.block.lastHash = null;
      return;
    }

    peer.destroy();
  }

  timer = setTimeout(destroy, this.load.timeout);

  peer.on('block', function(block) {
    self._handleBlock(block, peer);
  });

  // Split blocks and request them using multiple peers
  peer.on('blocks', function(hashes) {
    var i, hash;

    self.emit('blocks', hashes);

    if (hashes.length === 0) {
      // Reset global load
      self.block.lastHash = null;
      return;
    }

    if (!self.options.fullNode) {
      // Request each block
      hashes.forEach(function(hash) {
        self._request('filtered', hash);
      });

      self._scheduleRequests();

      // The part of the response is in chain, no need to escalate requests
      async.every(hashes, function(hash, cb) {
        self.chain.has(utils.toHex(hash), function(res) {
          cb(!res);
        });
      }, function(allNew) {
        if (!allNew) {
          self.block.lastHash = null;
          return;
        }

        // Store last hash to continue global load
        self.block.lastHash = hashes[hashes.length - 1];

        clearTimeout(timer);

        // Reinstantiate timeout
        if (self._load())
          timer = setTimeout(destroy, self.load.timeout);
      });

      return;
    }

    for (i = 0; i < hashes.length; i++) {
      hash = hashes[i];
      if (self.chain.hasOrphan(hash)) {
        peer.loadBlocks(self.chain.locatorHashes(), self.chain.getOrphanRoot(hash));
        continue;
      }

      if (i === hashes.length - 1)
        peer.loadBlocks(self.chain.locatorHashes(), 0);

      if (!self.chain.hasBlock(hash))
        self._request('block', hash);
    }

    self._scheduleRequests();

    self.block.lastHash = hashes[hashes.length - 1];

    clearInterval(interval);
    interval = setInterval(load, self.load.interval);

    clearTimeout(timer);
    timer = setTimeout(destroy, self.load.timeout);
  });
};

Pool.prototype._handleMerkle = function _handleMerkle(block, peer) {
  this._response(block);
  this.chain.add(block);
  this.emit('chain-progress', this.chain.fillPercent(), peer);
  this.emit('block', block, peer);
};

Pool.prototype._handleBlock = function _handleBlock(block, peer) {
  var chainIndex = this.chain.index;
  var hash, len, orphan, err;

  backoff = 0;

  this._response(block);

  hash = block.hash('hex');
  len = chainIndex.hashes.length;
  orphan = this.chain.hasOrphan(block);

  err = this.chain.add(block);
  if (err)
    this.emit('chain-error', err, peer);

  this.emit('_block', block, peer);

  if (this.chain.hasOrphan(block)) {
    peer.loadBlocks(this.chain.locatorHashes(), this.chain.getOrphanRoot(block));
    if (!orphan)
      this.emit('orphan', block, peer);
    return;
  }

  if (chainIndex.hashes.length === len)
    return;

  this.emit('chain-progress', this.chain.fillPercent(), peer);
  this.emit('block', block, peer);
};

Pool.prototype.isFull = function isFull() {
  return this.chain.isFull();
};

Pool.prototype._loadRange = function _loadRange(hashes, force) {
  var now = +new Date();
  var last;

  if (this.options.fullNode)
    return;

  if (!hashes)
    return;

  if (hashes.length <= 1)
    return;

  // Limit number of requests
  if (!force && now - this.load.lastRange < this.load.rangeWindow)
    return;

  this.load.lastRange = now;

  if (!this.peers.load)
    this._addLoader();

  last = hashes[hashes.length - 1];

  hashes.slice(0, -1).forEach(function(hash) {
    this.peers.load.loadBlocks([ hash ], last);
  }, this);
};

Pool.prototype._load = function _load() {
  var self = this;
  var next;

  if (this.request.queue.length >= this.load.hwm) {
    this.load.hiReached = true;
    return false;
  }

  this.load.hiReached = false;

  console.log(
    'Requesting inv packet from %s with getblocks',
    this.peers.load.address);

  if (!this.options.fullNode) {
    next = function(hash) {
      if (!self.peers.load)
        self._addLoader();
      else
        self.peers.load.loadBlocks([ hash ]);
    };

    // Load more blocks, starting from last hash
    if (this.block.lastHash)
      next(this.block.lastHash);
    else
      this.chain.getLast(next);

    return true;
  }

  if (!this.peers.load)
    this._addLoader();
  else
    this.peers.load.loadBlocks(this.chain.locatorHashes(), 0);

  return true;
};

Pool.prototype._addPeer = function _addPeer(backoff) {
  var self = this;
  var peer;

  if (this.destroyed)
    return;

  if (this.peers.block.length + this.peers.pending.length >= this.size)
    return;

  peer = new bcoin.peer(this, this.createSocket, {
    backoff: backoff,
    startHeight: this.options.startHeight,
    relay: this.options.relay
  });

  this.peers.pending.push(peer);

  peer._retry = 0;

  // Create new peer on failure
  peer.on('error', function(err) {
    self.emit('error', err, peer);
  });

  peer.once('close', function() {
    self._removePeer(peer);
    if (self.destroyed)
      return;
    self._addPeer(Math.max(backoff + self.backoff.delta, self.backoff.max));
  });

  peer.once('ack', function() {
    var i;

    if (self.destroyed)
      return;

    i = self.peers.pending.indexOf(peer);
    if (i !== -1) {
      self.peers.pending.splice(i, 1);
      self.peers.block.push(peer);
    }

    peer.updateWatch();

    self.tx.list.forEach(function(entry) {
      var result = peer.broadcast(entry.tx);
      if (!result) return;
      result[0].once('request', function() {
        entry.e.emit('ack', peer);
      });
    });

    self._scheduleRequests();
  });

  if (!this.options.fullNode) {
    peer.on('merkleblock', function(block) {
      // Reset backoff, peer seems to be responsive
      backoff = 0;
      self._handleMerkle(block, peer);
    });
  } else {
    peer.on('block', function(block) {
      // Reset backoff, peer seems to be responsive
      backoff = 0;
      self._handleBlock(block, peer);
    });
  }

  // Just FYI
  peer.on('reject', function(payload) {
    self.emit('reject', payload, peer);
  });

  peer.on('notfound', function(items) {
    items.forEach(function(item) {
      var req = self.request.map[utils.toHex(item.hash)];
      if (req)
        req.finish(null);
    });
  });

  peer.on('tx', function(tx) {
    self._response(tx);
    self.emit('tx', tx, peer);
  });

  peer.on('addr', function(addr) {
    self.emit('addr', addr, peer);
  });

  peer.on('blocks', function(blocks) {
    if (blocks.length === 1)
      self.block.bestHash = peer.bestBlock;
    self.emit('blocks', blocks, peer);
  });

  peer.on('txs', function(txs) {
    self.emit('txs', txs, peer);
  });

  peer.on('version', function(version) {
    if (version.height > self.block.bestHeight)
      self.block.bestHeight = version.height;
    self.emit('version', version, peer);
  });

  utils.nextTick(function() {
    self.emit('peer', peer);
  });
};

Pool.prototype.bestPeer = function bestPeer() {
  return this.peers.block.reduce(function(best, peer) {
    if (!peer.version || !peer.socket)
      return;

    if (!best || peer.version.height > best.version.height)
      return peer;

    return best;
  }, null);
};

Pool.prototype._removePeer = function _removePeer(peer) {
  var i = this.peers.pending.indexOf(peer);
  if (i !== -1)
    this.peers.pending.splice(i, 1);

  i = this.peers.block.indexOf(peer);
  if (i !== -1)
    this.peers.block.splice(i, 1);

  if (this.peers.load === peer)
    this.peers.load = null;
};

Pool.prototype.watch = function watch(id) {
  var hid, i;

  if (this.options.fullNode)
    return;

  if (id instanceof bcoin.wallet) {
    this.watchWallet(id);
    return;
  }

  if (id) {
    hid = utils.toHex(id);
    if (this.watchMap[hid])
      this.watchMap[hid]++;
    else
      this.watchMap[hid] = 1;

    if (this.bloom.test(id, 'hex'))
      return;

    this.bloom.add(id, 'hex');
  }

  if (this.peers.load)
    this.peers.load.updateWatch();

  for (i = 0; i < this.peers.block.length; i++)
    this.peers.block[i].updateWatch();
};

Pool.prototype.unwatch = function unwatch(id) {
  var i;

  if (this.options.fullNode)
    return;

  id = utils.toHex(id);

  if (!this.bloom.test(id, 'hex'))
    return;

  if (!this.watchMap[id] || --this.watchMap[id] !== 0)
    return;

  delete this.watchMap[id];

  // Reset bloom filter
  this.bloom.reset();
  Object.keys(this.watchMap).forEach(function(id) {
    this.bloom.add(id, 'hex');
  }, this);

  // Resend it to peers
  if (this.peers.load)
    this.peers.load.updateWatch();

  for (i = 0; i < this.peers.block.length; i++)
    this.peers.block[i].updateWatch();
};

Pool.prototype.addWallet = function addWallet(w, defaultTs) {
  var self = this;
  var e;

  if (this.wallets.indexOf(w) !== -1)
    return false;

  this.watchWallet(w);
  this.wallets.push(w);

  e = new EventEmitter();

  if (w.loaded)
    search(w.lastTs);
  else
    w.once('load', function(lastTs) { search(w.lastTs) });

  function search(ts) {
    // Relay pending TXs
    // NOTE: It is important to do it after search, because search could
    // add TS to pending TXs, thus making them confirmed
    w.pending().forEach(function(tx) {
      self.sendTX(tx);
    });

    // Search for last week by default
    if (!ts)
      ts = defaultTs || ((+new Date / 1000) - 7 * 24 * 3600);

    if (self.options.fullNode)
      return;

    self.search(false, ts, e);
  }

  return e;
};

Pool.prototype.removeWallet = function removeWallet(w) {
  var i = this.wallets.indexOf(w);
  if (i == -1)
    return;
  this.wallets.splice(i, 1);
  this.unwatchWallet(w);
};

Pool.prototype.watchWallet = function watchWallet(w) {
  if (this.options.fullNode)
    return;

  if (w.type === 'scripthash') {
    // For the redeem script hash in outputs:
    this.watch(w.getFullHash());
    // For the redeem script in inputs:
    this.watch(w.getFullPublicKey());
  }

  // For the pubkey hash in outputs:
  this.watch(w.getOwnHash());
  // For the pubkey in inputs:
  this.watch(w.getOwnPublicKey());
};

Pool.prototype.unwatchWallet = function unwatchWallet(w) {
  if (this.options.fullNode)
    return;

  if (w.type === 'scripthash') {
    // For the redeem script hash in p2sh outputs:
    this.unwatch(w.getFullHash());
    // For the redeem script in p2sh inputs:
    this.unwatch(w.getFullPublicKey());
  }

  // For the pubkey hash in p2pk/multisig outputs:
  this.unwatch(w.getOwnHash());
  // For the pubkey in p2pkh inputs:
  this.unwatch(w.getOwnPublicKey());
};

Pool.prototype.search = function search(id, range, e) {
  var self = this;

  if (this.options.fullNode)
    return;<commit_msg>allow watch() to be used on a full node.<commit_after>/**
 * pool.js - peer management for bcoin
 * Copyright (c) 2014-2015, <NAME> (MIT License)
 * https://github.com/indutny/bcoin
 */

var async = require('async');
var inherits = require('inherits');
var EventEmitter = require('events').EventEmitter;

var bcoin = require('../bcoin');
var utils = bcoin.utils;
var assert = utils.assert;
var network = bcoin.protocol.network;

/**
 * Pool
 */

function Pool(options) {
  var self = this;
  var Chain;

  if (!(this instanceof Pool))
    return new Pool(options);

  EventEmitter.call(this);

  this.options = options || {};

  if (this.options.network)
    network.set(this.options.network);

  this.options.fullNode = !!this.options.fullNode;
  this.options.relay == null
    ? (this.options.fullNode ? false : true)
    : this.options.relay;
  this.storage = this.options.storage;
  this.destroyed = false;
  this.size = options.size || 32;
  this.parallel = options.parallel || 2000;
  this.redundancy = options.redundancy || 2;

  this.backoff = {
    delta: options.backoffDelta || 500,
    max: options.backoffMax || 5000
  };

  this.load = {
    timeout: options.loadTimeout || 3000,
    interval: options.loadInterval || 5000,
    window: options.loadWindow || 250,
    lastRange: null,
    rangeWindow: options.rangeWindow || 1000,
    timer: null,
    lwm: options.lwm || this.parallel * 2,
    hwm: options.hwm || this.parallel * 8,
    hiReached: false
  };

  if (this.options.fullNode) {
    if (!options.loadTimeout)
      this.load.timeout = 30000;
    if (!options.loadInterval)
      this.load.interval = 5000;
  }

  this.maxRetries = options.maxRetries || 42;
  this.requestTimeout = options.requestTimeout || 10000;

  Chain = this.options.fullNode
    ? bcoin.fullChain
    : bcoin.spvChain;

  this.chain = new Chain({
    storage: this.storage,
    fullNode: this.options.fullNode,
    startHeight: this.options.startHeight
  });

  this.watchMap = {};

  this.bloom = new bcoin.bloom(
    8 * 1024,
    10,
    (Math.random() * 0xffffffff) | 0
  );

  this.peers = {
    // Peers that are loading blocks themselves
    block: [],
    // Peers that are still connecting
    pending: [],
    // Peers that are loading block ids
    load: null
  };

  this.block = {
    lastHash: null,
    bestHeight: 0,
    bestHash: null
  };

  this.request = {
    map: {},
    active: 0,
    queue: []
  };

  this.validate = {
    // 5 days scan delta for obtaining TXs
    delta: 5 * 24 * 3600,

    // Minimum verification depth
    minDepth: options.minValidateDepth || 0,

    // getTX map
    map: {}
  };

  // Currently broadcasted TXs
  this.tx = {
    list: [],
    timeout: options.txTimeout || 60000
  };

  // Added and watched wallets
  this.options.wallets = this.options.wallets || [];
  this.wallets = [];

  this.createSocket = options.createConnection || options.createSocket;
  assert(this.createSocket);

  this.chain.on('debug', function() {
    var args = Array.prototype.slice.call(arguments);
    self.emit.apply(self, ['debug'].concat(args));
  });

  if (!this.chain.loading) {
    this._init();
  } else {
    this.chain.once('load', function() {
      self._init();
    });
  }
}

inherits(Pool, EventEmitter);

Pool.prototype._init = function _init() {
  var self = this;
  var i;

  this._addLoader();

  for (i = 0; i < this.size; i++)
    this._addPeer(0);

  this._load();

  this.chain.on('missing', function(hash, preload, parent) {
    if (!self.options.fullNode) {
      self._request('filtered', hash, { force: true });
      self._scheduleRequests();
      self._loadRange(preload);
    }
  });

  this.chain.on('fork', function(height, hash, checkpoint) {
    var peer = self.peers.load;

    if (!self.options.fullNode)
      return;

    if (!peer)
      return;

    peer.destroy();
  });

  this.options.wallets.forEach(function(w) {
    self.addWallet(w);
  });
};

Pool.prototype._addLoader = function _addLoader() {
  var self = this;
  var peer, interval, timer;

  if (this.destroyed)
    return;

  if (this.peers.load != null)
    return;

  peer = new bcoin.peer(this, this.createSocket, {
    backoff: 750 * Math.random(),
    startHeight: this.options.startHeight,
    relay: this.options.relay
  });

  this.peers.load = peer;

  peer.on('error', function(err) {
    self.emit('error', err, peer);
  });

  peer.once('close', onclose);
  function onclose() {
    clearTimeout(timer);
    clearInterval(interval);
    self._removePeer(peer);
    if (self.destroyed)
      return;
    self._addLoader();
  }

  peer.once('ack', function() {
    peer.updateWatch();
    if (!self._load())
      clearTimeout(timer);
  });

  peer.on('version', function(version) {
    if (version.height > self.block.bestHeight)
      self.block.bestHeight = version.height;
    self.emit('version', version, peer);
  });

  function load() {
    self._load();
  }

  interval = setInterval(load, this.load.interval);

  function destroy() {
    // Chain is full and up-to-date
    if (self.chain.isFull()) {
      clearTimeout(timer);
      self.emit('full');
      self.block.lastHash = null;
      return;
    }

    peer.destroy();
  }

  timer = setTimeout(destroy, this.load.timeout);

  peer.on('block', function(block) {
    self._handleBlock(block, peer);
  });

  // Split blocks and request them using multiple peers
  peer.on('blocks', function(hashes) {
    var i, hash;

    self.emit('blocks', hashes);

    if (hashes.length === 0) {
      // Reset global load
      self.block.lastHash = null;
      return;
    }

    if (!self.options.fullNode) {
      // Request each block
      hashes.forEach(function(hash) {
        self._request('filtered', hash);
      });

      self._scheduleRequests();

      // The part of the response is in chain, no need to escalate requests
      async.every(hashes, function(hash, cb) {
        self.chain.has(utils.toHex(hash), function(res) {
          cb(!res);
        });
      }, function(allNew) {
        if (!allNew) {
          self.block.lastHash = null;
          return;
        }

        // Store last hash to continue global load
        self.block.lastHash = hashes[hashes.length - 1];

        clearTimeout(timer);

        // Reinstantiate timeout
        if (self._load())
          timer = setTimeout(destroy, self.load.timeout);
      });

      return;
    }

    for (i = 0; i < hashes.length; i++) {
      hash = hashes[i];
      if (self.chain.hasOrphan(hash)) {
        peer.loadBlocks(self.chain.locatorHashes(), self.chain.getOrphanRoot(hash));
        continue;
      }

      if (i === hashes.length - 1)
        peer.loadBlocks(self.chain.locatorHashes(), 0);

      if (!self.chain.hasBlock(hash))
        self._request('block', hash);
    }

    self._scheduleRequests();

    self.block.lastHash = hashes[hashes.length - 1];

    clearInterval(interval);
    interval = setInterval(load, self.load.interval);

    clearTimeout(timer);
    timer = setTimeout(destroy, self.load.timeout);
  });
};

Pool.prototype._handleMerkle = function _handleMerkle(block, peer) {
  this._response(block);
  this.chain.add(block);
  this.emit('chain-progress', this.chain.fillPercent(), peer);
  this.emit('block', block, peer);
};

Pool.prototype._handleBlock = function _handleBlock(block, peer) {
  var self = this;
  var chainIndex = this.chain.index;
  var hash, len, orphan, err;

  if (this.listeners('watched').length > 0) {
    utils.nextTick(function() {
      block.txs.forEach(function(tx) {
        if (self.isWatched(tx))
          self.emit('watched', tx, peer);
      });
    });
  }

  backoff = 0;

  this._response(block);

  hash = block.hash('hex');
  len = chainIndex.hashes.length;
  orphan = this.chain.hasOrphan(block);

  err = this.chain.add(block);
  if (err)
    this.emit('chain-error', err, peer);

  this.emit('_block', block, peer);

  if (this.chain.hasOrphan(block)) {
    peer.loadBlocks(this.chain.locatorHashes(), this.chain.getOrphanRoot(block));
    if (!orphan)
      this.emit('orphan', block, peer);
    return;
  }

  if (chainIndex.hashes.length === len)
    return;

  this.emit('chain-progress', this.chain.fillPercent(), peer);
  this.emit('block', block, peer);
};

Pool.prototype.isFull = function isFull() {
  return this.chain.isFull();
};

Pool.prototype._loadRange = function _loadRange(hashes, force) {
  var now = +new Date();
  var last;

  if (this.options.fullNode)
    return;

  if (!hashes)
    return;

  if (hashes.length <= 1)
    return;

  // Limit number of requests
  if (!force && now - this.load.lastRange < this.load.rangeWindow)
    return;

  this.load.lastRange = now;

  if (!this.peers.load)
    this._addLoader();

  last = hashes[hashes.length - 1];

  hashes.slice(0, -1).forEach(function(hash) {
    this.peers.load.loadBlocks([ hash ], last);
  }, this);
};

Pool.prototype._load = function _load() {
  var self = this;
  var next;

  if (this.request.queue.length >= this.load.hwm) {
    this.load.hiReached = true;
    return false;
  }

  this.load.hiReached = false;

  console.log(
    'Requesting inv packet from %s with getblocks',
    this.peers.load.address);

  if (!this.options.fullNode) {
    next = function(hash) {
      if (!self.peers.load)
        self._addLoader();
      else
        self.peers.load.loadBlocks([ hash ]);
    };

    // Load more blocks, starting from last hash
    if (this.block.lastHash)
      next(this.block.lastHash);
    else
      this.chain.getLast(next);

    return true;
  }

  if (!this.peers.load)
    this._addLoader();
  else
    this.peers.load.loadBlocks(this.chain.locatorHashes(), 0);

  return true;
};

Pool.prototype._addPeer = function _addPeer(backoff) {
  var self = this;
  var peer;

  if (this.destroyed)
    return;

  if (this.peers.block.length + this.peers.pending.length >= this.size)
    return;

  peer = new bcoin.peer(this, this.createSocket, {
    backoff: backoff,
    startHeight: this.options.startHeight,
    relay: this.options.relay
  });

  this.peers.pending.push(peer);

  peer._retry = 0;

  // Create new peer on failure
  peer.on('error', function(err) {
    self.emit('error', err, peer);
  });

  peer.once('close', function() {
    self._removePeer(peer);
    if (self.destroyed)
      return;
    self._addPeer(Math.max(backoff + self.backoff.delta, self.backoff.max));
  });

  peer.once('ack', function() {
    var i;

    if (self.destroyed)
      return;

    i = self.peers.pending.indexOf(peer);
    if (i !== -1) {
      self.peers.pending.splice(i, 1);
      self.peers.block.push(peer);
    }

    peer.updateWatch();

    self.tx.list.forEach(function(entry) {
      var result = peer.broadcast(entry.tx);
      if (!result) return;
      result[0].once('request', function() {
        entry.e.emit('ack', peer);
      });
    });

    self._scheduleRequests();
  });

  if (!this.options.fullNode) {
    peer.on('merkleblock', function(block) {
      // Reset backoff, peer seems to be responsive
      backoff = 0;
      self._handleMerkle(block, peer);
    });
  } else {
    peer.on('block', function(block) {
      // Reset backoff, peer seems to be responsive
      backoff = 0;
      self._handleBlock(block, peer);
    });
  }

  // Just FYI
  peer.on('reject', function(payload) {
    self.emit('reject', payload, peer);
  });

  peer.on('notfound', function(items) {
    items.forEach(function(item) {
      var req = self.request.map[utils.toHex(item.hash)];
      if (req)
        req.finish(null);
    });
  });

  peer.on('tx', function(tx) {
    self._response(tx);
    self.emit('tx', tx, peer);

    if (!self.options.fullNode)
      self.emit('watched', tx, peer);
  });

  peer.on('addr', function(addr) {
    self.emit('addr', addr, peer);
  });

  peer.on('blocks', function(blocks) {
    if (blocks.length === 1)
      self.block.bestHash = peer.bestBlock;
    self.emit('blocks', blocks, peer);
  });

  peer.on('txs', function(txs) {
    self.emit('txs', txs, peer);
  });

  peer.on('version', function(version) {
    if (version.height > self.block.bestHeight)
      self.block.bestHeight = version.height;
    self.emit('version', version, peer);
  });

  utils.nextTick(function() {
    self.emit('peer', peer);
  });
};

Pool.prototype.bestPeer = function bestPeer() {
  return this.peers.block.reduce(function(best, peer) {
    if (!peer.version || !peer.socket)
      return;

    if (!best || peer.version.height > best.version.height)
      return peer;

    return best;
  }, null);
};

Pool.prototype._removePeer = function _removePeer(peer) {
  var i = this.peers.pending.indexOf(peer);
  if (i !== -1)
    this.peers.pending.splice(i, 1);

  i = this.peers.block.indexOf(peer);
  if (i !== -1)
    this.peers.block.splice(i, 1);

  if (this.peers.load === peer)
    this.peers.load = null;
};

Pool.prototype.watch = function watch(id) {
  var hid, i;

  if (id instanceof bcoin.wallet) {
    this.watchWallet(id);
    return;
  }

  if (id) {
    hid = utils.toHex(id);
    if (this.watchMap[hid])
      this.watchMap[hid]++;
    else
      this.watchMap[hid] = 1;

    if (this.bloom.test(id, 'hex'))
      return;

    this.bloom.add(id, 'hex');
  }

  if (this.peers.load)
    this.peers.load.updateWatch();

  for (i = 0; i < this.peers.block.length; i++)
    this.peers.block[i].updateWatch();
};

Pool.prototype.unwatch = function unwatch(id) {
  var i;

  id = utils.toHex(id);

  if (!this.bloom.test(id, 'hex'))
    return;

  if (!this.watchMap[id] || --this.watchMap[id] !== 0)
    return;

  delete this.watchMap[id];

  // Reset bloom filter
  this.bloom.reset();
  Object.keys(this.watchMap).forEach(function(id) {
    this.bloom.add(id, 'hex');
  }, this);

  // Resend it to peers
  if (this.peers.load)
    this.peers.load.updateWatch();

  for (i = 0; i < this.peers.block.length; i++)
    this.peers.block[i].updateWatch();
};

// See "Filter matching algorithm":
// https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki
Pool.prototype.isWatched = function(tx, bloom) {
  var i, input, output, outHash;

  if (!bloom)
    bloom = this.bloom;

  function testScript(script) {
    return script.some(function(chunk) {
      if (!Array.isArray(chunk) || chunk.length === 0)
        return false;
      return bloom.test(chunk);
    });
  }

  // 1. Test the tx hash
  if (bloom.test(tx.hash()))
    return true;

  // 2. Test data elements in output scripts
  //    (may need to update filter on match)
  for (i = 0; i < tx.outputs.length; i++) {
    output = tx.outputs[i];
    // Test the output script
    if (testScript(output.script))
      return true;
  }

  // 3. Test prev_out structure
  // 4. Test data elements in input scripts
  for (i = 0; i < tx.inputs.length; i++) {
    input = tx.inputs[i];
    outHash = input.out.hash;

    if (typeof outHash === 'string')
      outHash = utils.toArray(outHash, 'hex');

    // Test the prev_out tx hash
    if (bloom.test(outHash))
      return true;

    // Test the prev_out script
    if (input.out.tx) {
      output = input.out.tx.outputs[input.out.index];
      if (testScript(output.script))
        return true;
    }

    // Test the input script
    if (testScript(input.script))
      return true;
  }

  // 5. No match
  return false;
};

Pool.prototype.addWallet = function addWallet(w, defaultTs) {
  var self = this;
  var e;

  if (this.wallets.indexOf(w) !== -1)
    return false;

  this.watchWallet(w);
  this.wallets.push(w);

  e = new EventEmitter();

  if (w.loaded)
    search(w.lastTs);
  else
    w.once('load', function() { search(w.lastTs) });

  function search(ts) {
    // Relay pending TXs
    // NOTE: It is important to do it after search, because search could
    // add TS to pending TXs, thus making them confirmed
    w.pending().forEach(function(tx) {
      self.sendTX(tx);
    });

    // Search for last week by default
    if (!ts)
      ts = defaultTs || ((+new Date / 1000) - 7 * 24 * 3600);

    if (self.options.fullNode)
      return;

    self.search(false, ts, e);
  }

  return e;
};

Pool.prototype.removeWallet = function removeWallet(w) {
  var i = this.wallets.indexOf(w);
  if (i == -1)
    return;
  this.wallets.splice(i, 1);
  this.unwatchWallet(w);
};

Pool.prototype.watchWallet = function watchWallet(w) {
  if (w.type === 'scripthash') {
    // For the redeem script hash in outputs:
    this.watch(w.getFullHash());
    // For the redeem script in inputs:
    this.watch(w.getFullPublicKey());
  }

  // For the pubkey hash in outputs:
  this.watch(w.getOwnHash());
  // For the pubkey in inputs:
  this.watch(w.getOwnPublicKey());
};

Pool.prototype.unwatchWallet = function unwatchWallet(w) {
  if (w.type === 'scripthash') {
    // For the redeem script hash in p2sh outputs:
    this.unwatch(w.getFullHash());
    // For the redeem script in p2sh inputs:
    this.unwatch(w.getFullPublicKey());
  }

  // For the pubkey hash in p2pk/multisig outputs:
  this.unwatch(w.getOwnHash());
  // For the pubkey in p2pkh inputs:
  this.unwatch(w.getOwnPublicKey());
};

Pool.prototype.search = function search(id, range, e) {
  var self = this;

  if (this.options.fullNode)
    return;
{'fraction_non_alphanumeric': 0.15737325081552392, 'fraction_numerical': 0.008322916984238286, 'mean_word_length': 2.750514520923851, 'pattern_counts': {'":': 1, '<': 19, '<?xml version=': 0, '>': 16, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4579667', 'n_tokens_mistral': 11587, 'n_tokens_neox': 11082, 'n_words': 2387}