<commit_before>'use strict'

var Map = require('es6-map')
var http = require('http')
var https = require('https')
var bl = require('bl')
var URL = require('url')
var steed = require('steed')()
var nextTick = require('process-nextick-args')
var protocols = {
  'https:': https,
  'http:': http
}

function deduplicator () {
  var queues = new Map()
  var each = steed.each
  var temporaries = new Map()

  return deduplicate

  function deduplicate (address, cb) {
    var options
    if (typeof address === 'string') {
      options = URL.parse(address)
    } else {
      options = address
      address = URL.format(address)
    }
    var temp = temporaries.get(address)
    if (temp) {
      nextTick(cb, null, temp, noop)
      return
    }
    var queue = queues.get(address)
    if (queue) {
      queue.push(cb)
      return
    } else {
      queues.set(address, [cb])
      // let's defer the setup
      // when we have some free cycles
      setImmediate(
        fillResult,
        protocols[options.protocol].get(options),
        address)
    }
  }

  function fillResult (req, address) {
    var result = bl(onData)
    result.address = address
    req.result = result
    req.on('response', autopipe)
  }

  function onData (err, data) {
    var address = this.address

    var queue = queues.get(address)
    queues.delete(address)

    temporaries.set(address, data)

    each(new State(err, data, address), queue, push, clear)<commit_msg>Inilining magic.<commit_after>'use strict'

var Map = require('es6-map')
var http = require('http')
var https = require('https')
var bl = require('bl')
var URL = require('url')
var steed = require('steed')()
var nextTick = require('process-nextick-args')
var protocols = {
  'https:': https,
  'http:': http
}

function deduplicator () {
  var queues = new Map()
  var each = steed.each
  var temporaries = new Map()

  return deduplicate

  function deduplicate (address, cb) {
    var options
    if (typeof address === 'string') {
      options = URL.parse(address)
    } else {
      options = address
      address = URL.format(address)
    }
    var temp = temporaries.get(address)
    if (temp) {
      nextTick(cb, null, temp, noop)
    } else {
      callMaybe(address, options, cb)
    }
  }

  function callMaybe (address, options, cb) {
    var queue = queues.get(address)
    if (queue) {
      queue.push(cb)
    } else {
      queues.set(address, [cb])
      // let's defer the setup
      // when we have some free cycles
      setImmediate(
        fillResult,
        protocols[options.protocol].get(options),
        address)
    }
  }

  function fillResult (req, address) {
    var result = bl(onData)
    result.address = address
    req.result = result
    req.on('response', autopipe)
  }

  function onData (err, data) {
    var address = this.address

    var queue = queues.get(address)
    queues.delete(address)

    temporaries.set(address, data)

    each(new State(err, data, address), queue, push, clear)
{'fraction_non_alphanumeric': 0.127212020033389, 'fraction_numerical': 0.000667779632721202, 'mean_word_length': 2.7217391304347824, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1402427', 'n_tokens_mistral': 973, 'n_tokens_neox': 946, 'n_words': 263}