<commit_before>// TODO: dhtPort and torrentPort should be consistent between restarts
// TODO: peerId and nodeId should be consistent between restarts

module.exports = WebTorrent

var createTorrent = require('create-torrent')
var debug = require('debug')('webtorrent')
var DHT = require('bittorrent-dht/client') // browser exclude
var EventEmitter = require('events').EventEmitter
var extend = require('xtend')
var hat = require('hat')
var inherits = require('inherits')
var loadIPSet = require('load-ip-set') // browser exclude
var parallel = require('run-parallel')
var parseTorrent = require('parse-torrent')
var speedometer = require('speedometer')
var zeroFill = require('zero-fill')

var FSStorage = require('./lib/fs-storage') // browser exclude
var Storage = require('./lib/storage')
var Torrent = require('./lib/torrent')

inherits(WebTorrent, EventEmitter)

/**
 * BitTorrent client version string (used in peer ID).
 * Generated from package.json major and minor version. For example:
 *   '0.16.1' -> '0016'
 *   '1.2.5' -> '0102'
 */
var VERSION = (require('./package.json').version || '__VERSION__')
  .match(/([0-9]+)/g).slice(0, 2).map(zeroFill(2)).join('')

/**
 * WebTorrent Client
 * @param {Object} opts
 */
function WebTorrent (opts) {
  var self = this
  if (!(self instanceof WebTorrent)) return new WebTorrent(opts)
  if (!opts) opts = {}
  EventEmitter.call(self)
  if (!debug.enabled) self.setMaxListeners(0)

  self.destroyed = false
  self.torrentPort = opts.torrentPort || 0
  self.tracker = opts.tracker !== undefined ? opts.tracker : true
  self.rtcConfig = opts.rtcConfig

  self.torrents = []

  self.downloadSpeed = speedometer()
  self.uploadSpeed = speedometer()

  self.storage = typeof opts.storage === 'function'
    ? opts.storage
    : (opts.storage !== false && typeof FSStorage === 'function' /* browser exclude */)
      ? FSStorage
      : Storage

  self.peerId = opts.peerId === undefined
    ? new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8')
    : typeof opts.peerId === 'string'
      ? new Buffer(opts.peerId, 'hex')
      : opts.peerId
  self.peerIdHex = self.peerId.toString('hex')

  self.nodeId = opts.nodeId === undefined
    ? new Buffer(hat(160), 'hex')
    : typeof opts.nodeId === 'string'
      ? new Buffer(opts.nodeId, 'hex')
      : opts.nodeId
  self.nodeIdHex = self.nodeId.toString('hex')

  // TODO: implement webtorrent-dht
  if (opts.dht !== false && typeof DHT === 'function' /* browser exclude */) {
    // use a single DHT instance for all torrents, so the routing table can be reused
    self.dht = new DHT(extend({ nodeId: self.nodeId }, opts.dht))
    self.dht.listen(opts.dhtPort)
  }

  debug('new webtorrent (peerId %s, nodeId %s)', self.peerIdHex, self.nodeIdHex)

  if (typeof loadIPSet === 'function') {
    loadIPSet(opts.blocklist, {
      headers: { 'user-agent': 'WebTorrent (http://webtorrent.io)' }
    }, function (err, ipSet) {
      if (err) return self.error('failed to load blocklist: ' + err.message)
      self.blocked = ipSet
      ready()
    })
  } else process.nextTick(ready)

  function ready () {
    if (self.destroyed) return
    self.ready = true
    self.emit('ready')
  }
}

/**
 * Seed ratio for all torrents in the client.
 * @type {number}
 */
Object.defineProperty(WebTorrent.prototype, 'ratio', {
  get: function () {
    var self = this
    var uploaded = self.torrents.reduce(function (total, torrent) {
      return total + torrent.uploaded
    }, 0)
    var downloaded = self.torrents.reduce(function (total, torrent) {
      return total + torrent.downloaded
    }, 0) || 1
    return uploaded / downloaded
  }
})

/**
 * Returns the torrent with the given `torrentId`. Convenience method. Easier than
 * searching through the `client.torrents` array. Returns `null` if no matching torrent
 * found.
 *
 * @param  {string|Buffer|Object} torrentId
 * @return {Torrent|null}
 */
WebTorrent.prototype.get = function (torrentId) {
  var self = this
  var parsed = parseTorrent(torrentId)
  if (!parsed.infoHash) throw new Error('Invalid torrent identifier')
  for (var i = 0, len = self.torrents.length; i < len; i++) {
    var torrent = self.torrents[i]
    if (torrent.infoHash === parsed.infoHash) return torrent
  }
  return null
}

/**
 * Start downloading a new torrent. Aliased as `client.download`.
 * @param {string|Buffer|Object} torrentId
 * @param {Object} opts torrent-specific options
 * @param {function=} ontorrent called when the torrent is ready (has metadata)
 */
WebTorrent.prototype.add =
WebTorrent.prototype.download = function (torrentId, opts, ontorrent) {
  var self = this
  if (self.destroyed) throw new Error('client is destroyed')
  debug('add %s', torrentId)
  if (typeof opts === 'function') {
    ontorrent = opts
    opts = {}
  }
  if (!opts) opts = {}

  opts.client = self
  opts.storage = opts.storage || self.storage

  if (!opts.storageOpts) opts.storageOpts = {}
  if (opts.tmp) opts.storageOpts.tmp = opts.tmp

  var torrent = new Torrent(torrentId, opts)
  self.torrents.push(torrent)

  function clientOnTorrent (_torrent) {
    if (torrent.infoHash === _torrent.infoHash) {
      ontorrent(torrent)
      self.removeListener('torrent', clientOnTorrent)
    }
  }
  if (ontorrent) self.on('torrent', clientOnTorrent)

  torrent.on('error', function (err) {
    self.emit('error', err, torrent)
  })

  torrent.on('listening', function (port) {
    self.emit('listening', port, torrent)
  })

  torrent.on('ready', function () {
    // Emit 'torrent' when a torrent is ready to be used
    debug('torrent')
    self.emit('torrent', torrent)
  })

  return torrent
}

/**
 * Start seeding a new file/folder.
 * @param  {string|File|FileList|Buffer|Array.<string|File|Buffer>} input
 * @param  {Object} opts
 * @param  {function} onseed
 */
WebTorrent.prototype.seed = function (input, opts, onseed) {
  var self = this
  if (self.destroyed) throw new Error('client is destroyed')
  debug('seed %s', input)
  if (typeof opts === 'function') {
    onseed = opts
    opts = {}
  }
  if (!opts) opts = {}
  if (!opts.storageOpts) opts.storageOpts = {}
  opts.storageOpts.noVerify = true

  createTorrent.parseInput(input, opts, function (err, files) {
    if (err) return self.emit('error', err)
    var streams = files.map(function (file) { return file.getStream })

    createTorrent(input, opts, function (err, torrentBuf) {
      if (err) return self.emit('error', err)

      // if client was destroyed asyncronously, bail early (or `add` will throw)
      if (self.destroyed) return

      self.add(torrentBuf, opts, function (torrent) {
        var tasks = [function (cb) {
          torrent.storage.load(streams, cb)
        }]
        if (self.dht) {
          tasks.push(function (cb) {
            torrent.on('dhtAnnounce', cb)
          })
        }
        parallel(tasks, function (err) {
          if (err) return self.emit('error', err)
          if (onseed) onseed(torrent)
          self.emit('seed', torrent)
        })
      })
    })
  })
}

/**
 * Remove a torrent from the client.
 * @param  {string|Buffer}   torrentId
 * @param  {function} cb
 */
WebTorrent.prototype.remove = function (torrentId, cb) {
  var self = this
  var torrent = self.get(torrentId)
  if (!torrent) throw new Error('No torrent with id ' + torrentId)
  debug('remove')
  self.torrents.splice(self.torrents.indexOf(torrent), 1)
  torrent.destroy(cb)
}

/**
 * Destroy the client, including all torrents and connections to peers.
 * @param  {function} cb
 */
WebTorrent.prototype.destroy = function (cb) {
  var self = this
  self.destroyed = true
  debug('destroy')

  var tasks = self.torrents.map(function (torrent) {
    return function (cb) {
      self.remove(torrent.infoHash, cb)
    }
  })

  if (self.dht) {
    tasks.push(function (cb) {
      self.dht.destroy(cb)
    })
  }

  parallel(tasks, cb)
}
<commit_msg>better debug logs<commit_after>// TODO: dhtPort and torrentPort should be consistent between restarts
// TODO: peerId and nodeId should be consistent between restarts

module.exports = WebTorrent

var createTorrent = require('create-torrent')
var debug = require('debug')('webtorrent')
var DHT = require('bittorrent-dht/client') // browser exclude
var EventEmitter = require('events').EventEmitter
var extend = require('xtend')
var hat = require('hat')
var inherits = require('inherits')
var loadIPSet = require('load-ip-set') // browser exclude
var parallel = require('run-parallel')
var parseTorrent = require('parse-torrent')
var speedometer = require('speedometer')
var zeroFill = require('zero-fill')

var FSStorage = require('./lib/fs-storage') // browser exclude
var Storage = require('./lib/storage')
var Torrent = require('./lib/torrent')

inherits(WebTorrent, EventEmitter)

/**
 * BitTorrent client version string (used in peer ID).
 * Generated from package.json major and minor version. For example:
 *   '0.16.1' -> '0016'
 *   '1.2.5' -> '0102'
 */
var VERSION = (require('./package.json').version || '__VERSION__')
  .match(/([0-9]+)/g).slice(0, 2).map(zeroFill(2)).join('')

/**
 * WebTorrent Client
 * @param {Object} opts
 */
function WebTorrent (opts) {
  var self = this
  if (!(self instanceof WebTorrent)) return new WebTorrent(opts)
  if (!opts) opts = {}
  EventEmitter.call(self)
  if (!debug.enabled) self.setMaxListeners(0)

  self.destroyed = false
  self.torrentPort = opts.torrentPort || 0
  self.tracker = opts.tracker !== undefined ? opts.tracker : true
  self.rtcConfig = opts.rtcConfig

  self.torrents = []

  self.downloadSpeed = speedometer()
  self.uploadSpeed = speedometer()

  self.storage = typeof opts.storage === 'function'
    ? opts.storage
    : (opts.storage !== false && typeof FSStorage === 'function' /* browser exclude */)
      ? FSStorage
      : Storage

  self.peerId = opts.peerId === undefined
    ? new Buffer('-WW' + VERSION + '-' + hat(48), 'utf8')
    : typeof opts.peerId === 'string'
      ? new Buffer(opts.peerId, 'hex')
      : opts.peerId
  self.peerIdHex = self.peerId.toString('hex')

  self.nodeId = opts.nodeId === undefined
    ? new Buffer(hat(160), 'hex')
    : typeof opts.nodeId === 'string'
      ? new Buffer(opts.nodeId, 'hex')
      : opts.nodeId
  self.nodeIdHex = self.nodeId.toString('hex')

  // TODO: implement webtorrent-dht
  if (opts.dht !== false && typeof DHT === 'function' /* browser exclude */) {
    // use a single DHT instance for all torrents, so the routing table can be reused
    self.dht = new DHT(extend({ nodeId: self.nodeId }, opts.dht))
    self.dht.listen(opts.dhtPort)
  }

  debug('new webtorrent (peerId %s, nodeId %s)', self.peerIdHex, self.nodeIdHex)

  if (typeof loadIPSet === 'function') {
    loadIPSet(opts.blocklist, {
      headers: { 'user-agent': 'WebTorrent (http://webtorrent.io)' }
    }, function (err, ipSet) {
      if (err) return self.error('failed to load blocklist: ' + err.message)
      self.blocked = ipSet
      ready()
    })
  } else process.nextTick(ready)

  function ready () {
    if (self.destroyed) return
    self.ready = true
    self.emit('ready')
  }
}

/**
 * Seed ratio for all torrents in the client.
 * @type {number}
 */
Object.defineProperty(WebTorrent.prototype, 'ratio', {
  get: function () {
    var self = this
    var uploaded = self.torrents.reduce(function (total, torrent) {
      return total + torrent.uploaded
    }, 0)
    var downloaded = self.torrents.reduce(function (total, torrent) {
      return total + torrent.downloaded
    }, 0) || 1
    return uploaded / downloaded
  }
})

/**
 * Returns the torrent with the given `torrentId`. Convenience method. Easier than
 * searching through the `client.torrents` array. Returns `null` if no matching torrent
 * found.
 *
 * @param  {string|Buffer|Object} torrentId
 * @return {Torrent|null}
 */
WebTorrent.prototype.get = function (torrentId) {
  var self = this
  var parsed = parseTorrent(torrentId)
  if (!parsed.infoHash) throw new Error('Invalid torrent identifier')
  for (var i = 0, len = self.torrents.length; i < len; i++) {
    var torrent = self.torrents[i]
    if (torrent.infoHash === parsed.infoHash) return torrent
  }
  return null
}

/**
 * Start downloading a new torrent. Aliased as `client.download`.
 * @param {string|Buffer|Object} torrentId
 * @param {Object} opts torrent-specific options
 * @param {function=} ontorrent called when the torrent is ready (has metadata)
 */
WebTorrent.prototype.add =
WebTorrent.prototype.download = function (torrentId, opts, ontorrent) {
  var self = this
  if (self.destroyed) throw new Error('client is destroyed')
  debug('add')
  if (typeof opts === 'function') {
    ontorrent = opts
    opts = {}
  }
  if (!opts) opts = {}

  opts.client = self
  opts.storage = opts.storage || self.storage

  if (!opts.storageOpts) opts.storageOpts = {}
  if (opts.tmp) opts.storageOpts.tmp = opts.tmp

  var torrent = new Torrent(torrentId, opts)
  self.torrents.push(torrent)

  function clientOnTorrent (_torrent) {
    if (torrent.infoHash === _torrent.infoHash) {
      ontorrent(torrent)
      self.removeListener('torrent', clientOnTorrent)
    }
  }
  if (ontorrent) self.on('torrent', clientOnTorrent)

  torrent.on('error', function (err) {
    self.emit('error', err, torrent)
  })

  torrent.on('listening', function (port) {
    self.emit('listening', port, torrent)
  })

  torrent.on('ready', function () {
    // Emit 'torrent' when a torrent is ready to be used
    debug('torrent')
    self.emit('torrent', torrent)
  })

  return torrent
}

/**
 * Start seeding a new file/folder.
 * @param  {string|File|FileList|Buffer|Array.<string|File|Buffer>} input
 * @param  {Object} opts
 * @param  {function} onseed
 */
WebTorrent.prototype.seed = function (input, opts, onseed) {
  var self = this
  if (self.destroyed) throw new Error('client is destroyed')
  debug('seed')
  if (typeof opts === 'function') {
    onseed = opts
    opts = {}
  }
  if (!opts) opts = {}
  if (!opts.storageOpts) opts.storageOpts = {}
  opts.storageOpts.noVerify = true

  createTorrent.parseInput(input, opts, function (err, files) {
    if (err) return self.emit('error', err)
    var streams = files.map(function (file) { return file.getStream })

    createTorrent(input, opts, function (err, torrentBuf) {
      if (err) return self.emit('error', err)

      // if client was destroyed asyncronously, bail early (or `add` will throw)
      if (self.destroyed) return

      self.add(torrentBuf, opts, function (torrent) {
        var tasks = [function (cb) {
          torrent.storage.load(streams, cb)
        }]
        if (self.dht) {
          tasks.push(function (cb) {
            torrent.on('dhtAnnounce', cb)
          })
        }
        parallel(tasks, function (err) {
          if (err) return self.emit('error', err)
          if (onseed) onseed(torrent)
          self.emit('seed', torrent)
        })
      })
    })
  })
}

/**
 * Remove a torrent from the client.
 * @param  {string|Buffer}   torrentId
 * @param  {function} cb
 */
WebTorrent.prototype.remove = function (torrentId, cb) {
  var self = this
  var torrent = self.get(torrentId)
  if (!torrent) throw new Error('No torrent with id ' + torrentId)
  debug('remove')
  self.torrents.splice(self.torrents.indexOf(torrent), 1)
  torrent.destroy(cb)
}

/**
 * Destroy the client, including all torrents and connections to peers.
 * @param  {function} cb
 */
WebTorrent.prototype.destroy = function (cb) {
  var self = this
  self.destroyed = true
  debug('destroy')

  var tasks = self.torrents.map(function (torrent) {
    return function (cb) {
      self.remove(torrent.infoHash, cb)
    }
  })

  if (self.dht) {
    tasks.push(function (cb) {
      self.dht.destroy(cb)
    })
  }

  parallel(tasks, cb)
}

{'fraction_non_alphanumeric': 0.14403763030765318, 'fraction_numerical': 0.004195270785659802, 'mean_word_length': 3.318693384573154, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2413671', 'n_tokens_mistral': 5224, 'n_tokens_neox': 5089, 'n_words': 1338}