<commit_before>
/**
 *  Read info from various AWS APIs and put the info into MongoDB.
 */

var sg                  = require('sgsg');
var _                   = sg._;
var jsMongo             = require('js-mongo');
var jsaws               = require('./jsaws');

var argvGet             = sg.argvGet;
var argvExtract         = sg.argvExtract;
var setOnMulti          = sg.setOnMulti;
var setOn               = sg.setOn;
var deref               = sg.deref;
var die                 = sg.die;
var getColl             = jsMongo.collection;
var eachDoc             = jsMongo.eachDoc;
var toArray             = jsMongo.toArray;

var dbHost              = '10.11.21.220';
var dbName              = 'aws_cluster';
var dbUrl               = 'mongodb://'+dbHost+':27017/'+dbName;

var collNames           = ['images', 'instances', 'vpcs', 'resourceRecordSets', 'addresses'];

var libCim = {};

libCim.clusterInMongo = function(argv, context, callback) {
  argv              = jsaws.prep(argv);
  var awsEc2        = jsaws.getEc2(argv);
  var awsSns        = jsaws.getSns(argv);
  var awsIam        = jsaws.getIam(argv);
  var region        = argvExtract(argv, 'region');
  var awsRoute53    = jsaws.getRoute53(argv);

  var count         = argvExtract(argv, 'count');
  var skipRemove    = argvExtract(argv, 'skip-remove');

  var stats         = {deleted:0};
  var colls         = {};

  one();
  function one() {

    return sg.__run([function(next) {
      // Remove all data
      return sg.__eachll(collNames, function(collName, nextColl) {
        return getColl(dbUrl, collName, function(err, collection) {

          colls[collName] = collection;

          if (skipRemove) { return nextColl(); }

          return collection.deleteMany({}, function(err, receipt) {
            log('deleteMany: '+collName, err, _.pick(receipt, 'result', 'deletedCount'));
            stats.deleted  += deref(receipt, 'deletedCount');
            return nextColl();
          });
        });
      }, next);

    }], function() {
      stats.inserts = {};
      return sg.__runll([function(next) {

        //==========================================================================================================
        //  Instances
        //==========================================================================================================

        // Do Instances -- fetch and then store
        return jsaws.get2Instances(argv, context, function(err, instances) {
          if (err || !instances || sg.numKeys(instances) === 0) { return next(); }

          return bulkInsert('instances', instances, next);
        });

      //==========================================================================================================
      //  Images
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Images(argv, context, function(err, images) {
          if (err || !images || sg.numKeys(images) === 0) { return next(); }

          return bulkInsert('images', images, next);
        });

      //==========================================================================================================
      //  Vpcs
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Vpcs(argv, context, function(err, vpcs) {
          if (err || !vpcs || sg.numKeys(vpcs) === 0) { return next(); }

          return bulkInsert('vpcs', vpcs, next);
        });

      //==========================================================================================================
      //  ResourceRecordSets
      //==========================================================================================================

      }, function(next) {

        var params = {
          zone_name   : "mobiledevprint.net.",
          zone2_name  : "mobilewebprint.net."
        };

        return jsaws.list2ResourceRecordSets(params, context, function(err, rrs) {
          if (err || !rrs || sg.numKeys(rrs) === 0) { return next(); }

          return bulkInsert('resourceRecordSets', rrs.ResourceRecordSets, next);
        });

      //==========================================================================================================
      //  Addresses
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Addresses(argv, context, function(err, addresses) {
          if (err || !addresses || sg.numKeys(addresses) === 0) { return next(); }

          return bulkInsert('addresses', addresses, next);
        });

      }], function() {
        if (--count > 0)    { log("Again!--------"); return setTimeout(one, 20000);}

        jsMongo.close();
        return callback(null, stats);
      });
    });
  }

  function bulkInsert(collName, items_, callback) {
    var items       = _.map(items_, function(item) { return insertOne(item); });
    return colls[collName].bulkWrite(items, function(err, receipt) {
      if (err) { return callback(err); }

      log(collName, err, _.pick(receipt, 'result', 'insertedCount'));
      stats.inserts[collName] = receipt.insertedCount;
<commit_msg>Put current state into DB<commit_after>
/**
 *  Read info from various AWS APIs and put the info into MongoDB.
 */

var sg                  = require('sgsg');
var _                   = sg._;
var jsMongo             = require('js-mongo');
var jsaws               = require('./jsaws');

var argvGet             = sg.argvGet;
var argvExtract         = sg.argvExtract;
var setOnMulti          = sg.setOnMulti;
var setOn               = sg.setOn;
var deref               = sg.deref;
var die                 = sg.die;
var getColl             = jsMongo.collection;
var eachDoc             = jsMongo.eachDoc;
var toArray             = jsMongo.toArray;
var findOneAndUpdate    = jsMongo.findOneAndUpdate;

var dbHost              = process.env.MARIO_RT_DB_IP || '10.11.21.220';
var dbName              = 'aws_cluster';
var dbUrl               = 'mongodb://'+dbHost+':27017/'+dbName;

var collNames           = ['images', 'instances', 'vpcs', 'resourceRecordSets', 'addresses'];

var libCim = {};

libCim.clusterInMongo = function(argv, context, callback) {
  argv              = jsaws.prep(argv);
  var awsEc2        = jsaws.getEc2(argv);
  var awsSns        = jsaws.getSns(argv);
  var awsIam        = jsaws.getIam(argv);
  var region        = argvExtract(argv, 'region');
  var awsRoute53    = jsaws.getRoute53(argv);

  var skipRemove    = argvExtract(argv, 'skip-remove');

  var stats         = {deleted:0, countsChange:0};
  var colls         = {};
  var dbStats;
  var cimStats;

  one();
  function one() {

    stats.countsChange = 0;

    return sg.__run([function(next) {
      // Get collections
      return sg.__eachll(collNames.concat(['dbStats']), function(collName, nextColl) {
        return getColl(dbUrl, collName, function(err, collection) {

          colls[collName] = collection;
          return nextColl();

        });

      }, function() {

        // Get the tracking stats, so we can put all the data we find in this run into the same group
        if (!(dbStats = colls.dbStats))   { return die('no dbStats collection', callback); }
        return findOneAndUpdate({inc:{building:1}}, {type:"clusterInMongo"}, dbStats, function(err, receipt) {
          if (!err && receipt && receipt.ok) {
            cimStats        = receipt.value;
            stats.building  = cimStats.building;
          }
          return next();
        });
      });

    // Run through the interesting aws-managed itmes, and put them into the group
    }, function(next) {
      stats.inserts = {};
      return sg.__runll([function(next) {

        //==========================================================================================================
        //  Instances
        //==========================================================================================================

        // Do Instances -- fetch and then store
        return jsaws.get2Instances(argv, context, function(err, instances) {
          if (err || !instances || sg.numKeys(instances) === 0) { return next(); }

          return bulkInsert('instances', instances, next);
        });

      //==========================================================================================================
      //  Images
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Images(argv, context, function(err, images) {
          if (err || !images || sg.numKeys(images) === 0) { return next(); }

          return bulkInsert('images', images, next);
        });

      //==========================================================================================================
      //  Vpcs
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Vpcs(argv, context, function(err, vpcs) {
          if (err || !vpcs || sg.numKeys(vpcs) === 0) { return next(); }

          return bulkInsert('vpcs', vpcs, next);
        });

      //==========================================================================================================
      //  ResourceRecordSets
      //==========================================================================================================

      }, function(next) {

        var params = {
          zone_name   : "mobiledevprint.net.",
          zone2_name  : "mobilewebprint.net."
        };

        return jsaws.list2ResourceRecordSets(params, context, function(err, rrs) {
          if (err || !rrs || sg.numKeys(rrs) === 0) { return next(); }

          return bulkInsert('resourceRecordSets', rrs.ResourceRecordSets, next);
        });

      //==========================================================================================================
      //  Addresses
      //==========================================================================================================

      }, function(next) {

        return jsaws.get2Addresses(argv, context, function(err, addresses) {
          if (err || !addresses || sg.numKeys(addresses) === 0) { return next(); }

          return bulkInsert('addresses', addresses, next);
        });

      }], next);

    // Now, set current to the group that we just built
    }, function(next) {
      return findOneAndUpdate({set:{current:cimStats.building}}, {type:"clusterInMongo"}, dbStats, function(err, receipt) {
        if (!err && receipt && receipt.ok) {
          cimStats        = receipt.value;
          stats.current   = cimStats.current;
        }
        return next();
      });

    }, function(next) {

      // Remove all data from old groups

      return sg.__eachll(collNames, function(collName, nextColl) {
        return getColl(dbUrl, collName, function(err, collection) {

          colls[collName] = collection;

          if (skipRemove) { return nextColl(); }

          return collection.deleteMany({group:{$lt: (cimStats.current - 2)}}, function(err, receipt) {
            log('deleteMany: '+collName, err, _.pick(receipt, 'result', 'deletedCount'));
            stats.deleted       += deref(receipt, 'deletedCount');
            stats.countsChange  -= deref(receipt, 'deletedCount');
            return nextColl();
          });
        });
      }, next);

    }], function() {
      return callback(null, stats);
    });
  }

  function bulkInsert(collName, items_, callback) {
    var items       = _.map(items_, function(item) { return insertOne(_.extend(item, {group:cimStats.building})); });
    return colls[collName].bulkWrite(items, function(err, receipt) {
      if (err) { return callback(err); }

      log(collName, err, _.pick(receipt, 'result', 'insertedCount'));
      stats.inserts[collName]   = receipt.insertedCount;
      stats.countsChange       += receipt.insertedCount;

{'fraction_non_alphanumeric': 0.29076733688961837, 'fraction_numerical': 0.006073040623717686, 'mean_word_length': 2.680459075807913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '196798', 'n_tokens_mistral': 3305, 'n_tokens_neox': 3054, 'n_words': 685}