<commit_before>var dns = require('dns');
var updateQueue = [];
var dnsCache = {};
var callbacks = {};
var reqCount = 10;
var fastReqCount = 20;
var TIMEOUT = 6000;
var UPDATE_INTERVAL = 1000;

function lookupDNS(hostname, callback) {
  var list = callbacks[hostname];
  if (list) {
    list.push(callback);
    return;
  }
  callbacks[hostname] = list = [callback];

  var done;
  function execCallback(err, ip) {
    var host;
    if (err) {
      host = dnsCache[hostname];
    } else {
      host = dnsCache[hostname] = {
        ip: ip,
        hostname: hostname,
        time: Date.now()
      };
    }
    addUpdateQueue(host);

    if (done) {
      return;
    }
    done = true;
    callbacks[hostname] = null;
    list.forEach(function(callback) {
      callback(err, ip);
    });
  }

  var timeout = setTimeout(function() {
    execCallback(new Error('Timeout'));
  }, TIMEOUT);

  try {
    dns.lookup(hostname, function (err, ip, type) {
      clearTimeout(timeout);
      err ? execCallback(err) :
execCallback(null, ip || (!type || type == 4 ? '127.0.0.1' : '0:0:0:0:0:0:0:1'));
    });
  } catch(err) {//如果断网，可能直接抛异常，https代理没有用到error-handler
    execCallback(err);
  }
}

function addUpdateQueue(host, fast) {
  if (!host) {
    return;
  }

  var len = updateQueue.length;
  if (len > 24000) {
    updateQueue.splice(7000, len - 12000);
    dnsCache = {};
    updateQueue.forEach(function(item) {
      dnsCache[item.hostname] = item;
    });
  }
  updateQueue[fast ? 'push' : 'unshift'](host);
  if (reqCount <= 0) {
    return;
  }

  --reqCount;
  if (fast) {
    updateQueue.pop();
    updateHost(host);
  } else {
    setTimeout(function() {
      var host = updateQueue.pop();
      if (host) {
        updateHost(host);
      } else {
        ++reqCount;
      }
    }, UPDATE_INTERVAL);
  }
}

function updateHost(host, fast) {
  lookupDNS(host.hostname, function(err, ip) {
    if (fast) {
      ++fastReqCount;
    } else {
      ++reqCount;
    }
  });
}

function fastUpdateHost(host) {
  if (Date.now() - host.time < TIMEOUT) {
    return;
  }

  var index = updateQueue.indexOf(host);
  if (index == -1) {
    return;
  }
  updateQueue.splice(index, 1);
  if (fastReqCount <= 0) {
    addUpdateQueue(host, true);
  } else {
    --fastReqCount;
    updateHost(host, true);
  }
}

module.exports = function lookup(hostname, callback, allowDnsCache) {
  var host = allowDnsCache ? dnsCache[hostname] : null;
  if (host) {
    callback(null, host.ip);
    fastUpdateHost(host);
    return host.ip;
  }
  lookupDNS(hostname, function(err, ip) {
    err ? lookupDNS(hostname, callback) : callback(err, ip);
  });
};<commit_msg>refactor: DNS Cache<commit_after>var dns = require('dns');
var noop = require('util').noop;

var dnsCache = {};
var callbacks = {};
var TIMEOUT = 5000;
var CACHE_TIME = 30000;

function lookupDNS(hostname, callback) {
  var list = callbacks[hostname];
  callback = callback || noop;
  if (list) {
    list.push(callback);
    return;
  }
  callbacks[hostname] = list = [callback];

  var done;
  function execCallback(err, ip) {
    if (!err) {
      dnsCache[hostname] = {
        ip: ip,
        hostname: hostname,
        time: Date.now()
      };
    }
    if (done) {
      return;
    }
    done = true;
    callbacks[hostname] = null;
    list.forEach(function(callback) {
      callback(err, ip);
    });
  }

  var timer = setTimeout(function() {
    execCallback(new Error('Timeout'));
  }, TIMEOUT);

  try {
    dns.lookup(hostname, function (err, ip, type) {
      clearTimeout(timer);
      if (err) {
        execCallback(err);
      } else {
        execCallback(null, ip || getDefaultIp(type));
      }
    });
  } catch(err) {//如果断网，可能直接抛异常，https代理没有用到error-handler
    execCallback(err);
  }
}

function getDefaultIp(type) {
  return !type || type == 4 ? '127.0.0.1' : '0:0:0:0:0:0:0:1';
}

module.exports = function lookup(hostname, callback, allowDnsCache) {
  var host = allowDnsCache ? dnsCache[hostname] : null;
  if (host) {
    callback(null, host.ip);
    if (Date.now() - host.time > CACHE_TIME) {
      lookupDNS(host.hostname);
    }
    return host.ip;
  }
  lookupDNS(hostname, function(err, ip) {
    err ? lookupDNS(hostname, callback) : callback(err, ip);
  });
};
{'fraction_non_alphanumeric': 0.15679360601786554, 'fraction_numerical': 0.01622002820874471, 'mean_word_length': 2.7788632326820606, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1727059', 'n_tokens_mistral': 1569, 'n_tokens_neox': 1545, 'n_words': 311}