<commit_before>'use strict';

var _ = require('lodash'),
    async = require('async'),
    config = require('../config'),
    dns = require('native-dns'),
    events = require('events'),
    ipaddr = require('ipaddr.js'),
    logger = require('../logger'),
    moment = require('moment'),
    redis = require('redis'),
    util = require('util');

var DnsResolver = function() {
  this.initialize.apply(this, arguments);
};

_.extend(DnsResolver, {
  NULL_ROUTE: '255.255.255.255',
  RECENT_IP_BUCKET_TIME_FORMAT: 'YYYY-MM-DDTHH',
});

util.inherits(DnsResolver, events.EventEmitter);
_.extend(DnsResolver.prototype, {
  resolved: {},

  initialize: function(configReloader, callback) {
    this.configReloader = configReloader;

    async.parallel([
      this.connectRedis.bind(this),
      this.fetchSystemNameServers.bind(this),
    ], this.finishInit.bind(this, callback));
  },

  connectRedis: function(asyncReadyCallback) {
    this.redis = redis.createClient(config.get('redis'));

    this.redis.on('error', function(error) {
      asyncReadyCallback(error);
    });

    this.redis.on('ready', function() {
      asyncReadyCallback(null);
    });
  },

  fetchSystemNameServers: function(asyncReadyCallback) {
    this.nameServers = dns.platform.name_servers;
    if(this.nameServers && this.nameServers.length > 0) {
      asyncReadyCallback(null);
    } else {
      dns.platform.once('ready', function() {
        this.nameServers = dns.platform.name_servers;
        asyncReadyCallback(null);
      }.bind(this));
    }
  },

  finishInit: function(callback, error) {
    logger.info('System DNS discovered: ', this.nameServers);

    // Watch for future changes to the system's DNS settings.
    dns.platform.watching = true;
    dns.platform.on('ready', this.handleNameServerChanges.bind(this));

    if(callback) {
      callback(error);
    }
  },

  handleNameServerChanges: function() {
    this.nameServers = dns.platform.name_servers;
    logger.info('System DNS changes detected. Using new name servers: ', this.nameServers);
  },

  resolveAllHosts: function(callback) {
    var apis = this.configReloader.apis;

    var hosts = [];
    apis.forEach(function(api) {
      if(api.servers) {
        hosts = hosts.concat(_.pluck(api.servers, 'host'));
      }
    });

    hosts = _.uniq(hosts);

    async.eachSeries(hosts, this.resolveHost.bind(this), this.handleResolveAllHosts.bind(this, callback));
  },

  handleResolveAllHosts: function(callback) {
    callback(null);
  },

  resolveHost: function(host, resolveHostCallback) {
    logger.debug('Resolving host: ', host);

    if(!this.resolved[host]) {
      this.resolved[host] = {
        ips: [],
        names: [],
      };
    }

    // If an IP address was given as the host, we don't need to do anything
    // else.
    if(ipaddr.isValid(host)) {
      _.merge(this.resolved[host], {
        ips: [host],
        existingIp: host,
        activeIp: host,
        ttl: null,
      });

      this.finishResolveHost(host, resolveHostCallback);
      return false;
    }

    async.series([
      this.fetchCachedHost.bind(this, host),
      this.sendHostQuery.bind(this, host),
      this.lookupFallback.bind(this, host),
      this.setActiveIp.bind(this, host),
    ], this.finishResolveHost.bind(this, host, resolveHostCallback));
  },

  fetchCachedHost: function(host, asyncCallback) {
    var resolved = this.resolved[host];
    if(resolved.activeIp) {
      resolved.existingIp = resolved.activeIp;
      resolved.activeIp = null;

      asyncCallback(null);
    } else {
      this.redis.get('router_active_ip:' + host, function(error, ip) {
        if(error) {
          logger.error('Error fetching active IP: ', error);
          asyncCallback(error);
        }

        if(ip) {
          resolved.existingIp = ip;
        }

        asyncCallback(null);
      }.bind(this));
    }
  },

  sendHostQuery: function(host, asyncCallback) {
    var question = dns.Question({
      name: host,
      type: 'A',
    });

    var request = dns.Request({
      question: question,
      server: this.nameServers[0],
      timeout: 5000,
    });

    request.on('message', this.handleResolveHostMessage.bind(this, host));
    request.on('end', asyncCallback);

    request.send();
  },

  handleResolveHostMessage: function(host, error, message) {
    if(error) {
      return false;
    }

    var resolved = this.resolved[host];

    message.answer.forEach(function(answer) {
      if(answer.address) {
        logger.debug('Resolved host: ', host, answer.name, answer.address);
        resolved.ips.push(answer.address);

        if(answer.name) {
          resolved.names.push(answer.name);
        }

        if(answer.ttl && !resolved.ttl) {
          resolved.ttl = answer.ttl;
        }
      }
    });

    if(!resolved.ttl) {
      resolved.ttl = 60 * 60; // 1 hour
    }

    // Extremely large TTLs probably won't occur, but if they do, they break
    // setTimeout, so set a cap: http://stackoverflow.com/a/3468650/222487
    if(resolved.ttl > 2000000) {
      resolved.ttl = 2000000;<commit_msg>Prevent super-frequent DNS polling (eg, every 1 second).<commit_after>'use strict';

var _ = require('lodash'),
    async = require('async'),
    config = require('../config'),
    dns = require('native-dns'),
    events = require('events'),
    ipaddr = require('ipaddr.js'),
    logger = require('../logger'),
    moment = require('moment'),
    redis = require('redis'),
    util = require('util');

var DnsResolver = function() {
  this.initialize.apply(this, arguments);
};

_.extend(DnsResolver, {
  NULL_ROUTE: '255.255.255.255',
  RECENT_IP_BUCKET_TIME_FORMAT: 'YYYY-MM-DDTHH',
});

util.inherits(DnsResolver, events.EventEmitter);
_.extend(DnsResolver.prototype, {
  resolved: {},

  initialize: function(configReloader, callback) {
    this.configReloader = configReloader;

    async.parallel([
      this.connectRedis.bind(this),
      this.fetchSystemNameServers.bind(this),
    ], this.finishInit.bind(this, callback));
  },

  connectRedis: function(asyncReadyCallback) {
    this.redis = redis.createClient(config.get('redis'));

    this.redis.on('error', function(error) {
      asyncReadyCallback(error);
    });

    this.redis.on('ready', function() {
      asyncReadyCallback(null);
    });
  },

  fetchSystemNameServers: function(asyncReadyCallback) {
    this.nameServers = dns.platform.name_servers;
    if(this.nameServers && this.nameServers.length > 0) {
      asyncReadyCallback(null);
    } else {
      dns.platform.once('ready', function() {
        this.nameServers = dns.platform.name_servers;
        asyncReadyCallback(null);
      }.bind(this));
    }
  },

  finishInit: function(callback, error) {
    logger.info('System DNS discovered: ', this.nameServers);

    // Watch for future changes to the system's DNS settings.
    dns.platform.watching = true;
    dns.platform.on('ready', this.handleNameServerChanges.bind(this));

    if(callback) {
      callback(error);
    }
  },

  handleNameServerChanges: function() {
    this.nameServers = dns.platform.name_servers;
    logger.info('System DNS changes detected. Using new name servers: ', this.nameServers);
  },

  resolveAllHosts: function(callback) {
    var apis = this.configReloader.apis;

    var hosts = [];
    apis.forEach(function(api) {
      if(api.servers) {
        hosts = hosts.concat(_.pluck(api.servers, 'host'));
      }
    });

    hosts = _.uniq(hosts);

    async.eachSeries(hosts, this.resolveHost.bind(this), this.handleResolveAllHosts.bind(this, callback));
  },

  handleResolveAllHosts: function(callback) {
    callback(null);
  },

  resolveHost: function(host, resolveHostCallback) {
    logger.debug('Resolving host: ', host);

    if(!this.resolved[host]) {
      this.resolved[host] = {
        ips: [],
        names: [],
      };
    }

    // If an IP address was given as the host, we don't need to do anything
    // else.
    if(ipaddr.isValid(host)) {
      _.merge(this.resolved[host], {
        ips: [host],
        existingIp: host,
        activeIp: host,
        ttl: null,
      });

      this.finishResolveHost(host, resolveHostCallback);
      return false;
    }

    async.series([
      this.fetchCachedHost.bind(this, host),
      this.sendHostQuery.bind(this, host),
      this.lookupFallback.bind(this, host),
      this.setActiveIp.bind(this, host),
    ], this.finishResolveHost.bind(this, host, resolveHostCallback));
  },

  fetchCachedHost: function(host, asyncCallback) {
    var resolved = this.resolved[host];
    if(resolved.activeIp) {
      resolved.existingIp = resolved.activeIp;
      resolved.activeIp = null;

      asyncCallback(null);
    } else {
      this.redis.get('router_active_ip:' + host, function(error, ip) {
        if(error) {
          logger.error('Error fetching active IP: ', error);
          asyncCallback(error);
        }

        if(ip) {
          resolved.existingIp = ip;
        }

        asyncCallback(null);
      }.bind(this));
    }
  },

  sendHostQuery: function(host, asyncCallback) {
    var question = dns.Question({
      name: host,
      type: 'A',
    });

    var request = dns.Request({
      question: question,
      server: this.nameServers[0],
      timeout: 5000,
    });

    request.on('message', this.handleResolveHostMessage.bind(this, host));
    request.on('end', asyncCallback);

    request.send();
  },

  handleResolveHostMessage: function(host, error, message) {
    if(error) {
      return false;
    }

    var resolved = this.resolved[host];

    message.answer.forEach(function(answer) {
      if(answer.address) {
        logger.debug('Resolved host: ', host, answer.name, answer.address);
        resolved.ips.push(answer.address);

        if(answer.name) {
          resolved.names.push(answer.name);
        }

        if(answer.ttl && !resolved.ttl) {
          resolved.ttl = answer.ttl;
        }
      }
    });

    if(!resolved.ttl) {
      resolved.ttl = 60 * 60; // 1 hour
    }

    // Extremely large TTLs probably won't occur, but if they do, they break
    // setTimeout, so set a cap: http://stackoverflow.com/a/3468650/222487
    if(resolved.ttl > 2000000) {
      resolved.ttl = 2000000;
    }

    // Sporadically we've seen TTLs being reported back as 1 second for certain
    // domains for a while. This causes unnecessary cpu load and quick dns
    // polling, so force a minimum ttl of 40 seconds.
    if(resolved.ttl < 40) {
      resolved.ttl = 40;
{'fraction_non_alphanumeric': 0.1421773886562619, 'fraction_numerical': 0.010277883517320136, 'mean_word_length': 2.7979761474521143, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4607752', 'n_tokens_mistral': 3436, 'n_tokens_neox': 3280, 'n_words': 613}