<commit_before>/*
  Copyright(c) 2012 meltmedia
  MIT LICENSE
 */
var _ = require('underscore'),
    step = require('step'),
    winston = require('winston');

_.str = require('underscore.string');

// Utils
function isDefined(obj) {
  if (typeof(obj) != 'undefined' && obj != null) {
    return true;
  }
  else {
    return false;
  }
};

var hosts = {
  ips: [],
  endpoints: [],
  home: null,
  whitelist: [],

  setWhitelist: function(whitelist) {
    if (_.isArray(whitelist)) {
      hosts.whitelist = whitelist;
    }
  },

  setHome: function(home) {
    hosts.home = home;
  },

  addEndpoint: function(endpoint) {
    hosts.endpoints.push(endpoint);
  },

  getAddress: function(instance) {
    try {
      if (_.str.startsWith(instance["placement"]["availabilityZone"], hosts.home)) {
        return instance["privateIpAddress"];
      }
      else {
        return instance["ipAddress"];
      }
    } catch (err) {
      winston.debug("error getting address: ", err);
      return null;
    }
  },

  getRegion: function(instance) {
    try {
      return instance["placement"]["availabilityZone"];
    } catch (err) {
      return null;
    }
  },

  allow: function(address) {
    if (_.indexOf(hosts.ips, address) != -1) {
      return true;
    }

    return false;
  },

  load: function(callback) {
    var addresses = [];
    step(
      function start() {
        hosts.whitelist.forEach(function(address) {
          addresses.push(address);
        });

        var group = this.group();
        hosts.endpoints.forEach(function (endpoint) {
          endpoint.call("DescribeInstances", {}, group());
        });
      },
      function parse(err, results) {
        if (err) {
          callback(err);
          return;
        }
        else {
          results.forEach(function(result) {
            var i = 0;
            var region = null;
            var acc = null;
          
            result["reservationSet"]["item"].forEach(function(instance) {
              if (region == null) {
                region = hosts.getRegion(instance["instancesSet"]["item"]).slice(0, -1);
                acc = instance["ownerId"];
              }

              var address = hosts.getAddress(instance["instancesSet"]["item"]);

              if (isDefined(address)) {
                i++;
                addresses.push(address);
              }
            });

            winston.debug("loaded " + i + " from " + acc + " " + region);
          });

          hosts.ips = addresses;
          callback(null, addresses);
        }
      }
    );
  }
};

module.exports = hosts;
<commit_msg>adding support for reservation sets<commit_after>var _ = require('underscore'),
    step = require('step'),
    winston = require('winston'),
    util = require('util');

_.str = require('underscore.string');

// Utils
function isDefined(obj) {
  if (typeof(obj) != 'undefined' && obj != null) {
    return true;
  }
  else {
    return false;
  }
};

var hosts = {
  ips: [],
  endpoints: [],
  home: null,
  whitelist: [],

  setWhitelist: function(whitelist) {
    if (_.isArray(whitelist)) {
      hosts.whitelist = whitelist;
    }
  },

  setHome: function(home) {
    hosts.home = home;
  },

  addEndpoint: function(endpoint) {
    hosts.endpoints.push(endpoint);
  },

  getAddress: function(instance) {
    try {
      if (_.str.startsWith(instance["placement"]["availabilityZone"], hosts.home)) {
        return instance["privateIpAddress"];
      }
      else {
        return instance["ipAddress"];
      }
    } catch (err) {
      winston.warn("error getting address for [" + instance + "]: ", err);
      winston.debug(util.inspect(instance, {showHidden: false, depth: null}));
      return null;
    }
  },

  getRegion: function(instance) {
    return instance["placement"]["availabilityZone"]
  },

  allow: function(address) {
    if (_.indexOf(hosts.ips, address) != -1) {
      return true;
    }

    return false;
  },

  load: function(callback) {
    var addresses = [];
    step(
      function start() {
        hosts.whitelist.forEach(function(address) {
          addresses.push(address);
        });

        var group = this.group();
        hosts.endpoints.forEach(function (endpoint) {
          endpoint.call("DescribeInstances", {}, group());
        });
      },
      function parse(err, results) {
        if (err) {
          callback(err);
          return;
        }
        else {
          results.forEach(function(result) {
            var i = 0;
            var region = null;
            var acc = null;

            result["reservationSet"]["item"].forEach(function(instance) {
              if (region == null) {
                region = hosts.getRegion(instance["instancesSet"]["item"]).slice(0, -1);
                acc = instance["ownerId"];
              }

              //winston.debug(util.inspect(instance, {showHidden: false, depth: null}));
              var set = null;
              if (!util.isArray(instance["instancesSet"]["item"])) {
                set = [instance["instancesSet"]["item"]];
              }
              else {
                set = instance["instancesSet"]["item"];
              }
              set.forEach(function(item) {
                var address = hosts.getAddress(item);

                if (isDefined(address)) {
                  i++;
                  addresses.push(address);
                }
              });
            });

            winston.debug("loaded " + i + " from " + acc + " " + region);
          });

          hosts.ips = addresses;
          callback(null, addresses);
        }
      }
    );
  }
};

module.exports = hosts;

{'fraction_non_alphanumeric': 0.14968152866242038, 'fraction_numerical': 0.0021231422505307855, 'mean_word_length': 1.8915601023017903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4279977', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1639, 'n_words': 310}