<commit_before>var MongoClient = require("mongodb").MongoClient;
var fs = require("fs");

var url = "mongodb://admin:<EMAIL>:45355/gcmt_dev?connectionTimeoutMS=360000000&socketTimeoutMS=360000000";

MongoClient.connect(url, function(err, db) {
  if (err) {
    console.log("ERROR");
    throw err;
  }

  var filenames = ["ATA", "gcmt_icons_reduced", "HimaTibetMap", "plate_bounds_bird02"];
  
  var startTime = process.hrtime()[0];

  db.collection("caitlinstestcollection").deleteMany({}, (err, res) => {
    console.log("Starting inserts");
    filenames.map((filename) => {
      var json = JSON.parse(fs.readFileSync("./data/" + filename + ".geojson", "utf8"));
      var features = json.features;
      console.log("Starting to insert " + filename);
      db.collection("caitlintestcollection").insertMany(features, (err, res) => {
        if (err) throw err;
        var totalTime = process.hrtime()[0] - startTime;
        console.log("All entries from " + filename + " inserted ok. Elapsed time is " + totalTime + " seconds.");
        /** Don't know why this part hangs
        db.collection("caitlintestcollection").count(function(err, res) {
            console.log("Total count is " + count);
            console.log("Insertion rate is " + (count/totalTime) + " records per second.");
        });
        **/
      });
    });
  })
});<commit_msg>Modify initial load script to support storing base64-encoded icon images in database<commit_after>var MongoClient = require("mongodb").MongoClient;
var fs = require("fs");

var url = "mongodb://admin:<EMAIL>:45355/gcmt_dev?connectionTimeoutMS=360000000&socketTimeoutMS=360000000";

MongoClient.connect(url, function(err, db) {
  if (err) {
    console.log("ERROR");
    throw err;
  }

//  var filenames = ["ATA", "gcmt_icons_reduced", "HimaTibetMap", "plate_bounds_bird02"];
  var filenames = ["gcmt_icons_reduced"];
  
  var startTime = process.hrtime()[0];

  db.collection("caitlintestcollection").deleteMany({}, (err, res) => {
    console.log("Starting inserts");

    filenames.map((filename) => {
      var json = JSON.parse(fs.readFileSync("./data/" + filename + ".geojson", "utf8"));

      var features = json.features;
      var base64iconUrl = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAB4AAAARCAYAAADKZhx3AAAMFGlDQ1BJQ0MgUHJvZmlsZQAASImVlwdUk8kWx+crKYSEFoiAlNCb9A7SexGQDjZCEiCUEAJBxY4sKrgWVESwoisgtrUAstiwl0XAXl8QUVlZFws2VN4kAfS5b887b86ZL7/cuffOfyYzX2YAUHRgCQTZqBIAOfwCYXSQLzMxKZlJEgME4IACtIEGi50v8ImKCgf/WN7dgt6wXLeU5Ppnv/9alDncfDYASBTkVE4+OwfyEQBwTbZAWAAAoRPaDWYXCCT8FrKqEAoEgEiWcLqMtSScKmMbqU9stB9kfwDIVBZLmA6AgiQ/s5CdDvMoCCDb8Dk8PuTtkD3ZGSwOZDHkSTk5uZAVqZBNU7/Lk/4fOVPHc7JY6eMsG4u0kP15+YJs1tz/czr+d8nJFo31oQ8rNUMYHC0ZM5y3+qzcMAlD7UgbPzUiErIK5Is8jtRfwvcyRMFxo/4D7Hw/OGeAAQAK<KEY>';
      features.map((feature) => {
        feature.properties.icon.iconUrl = base64iconUrl;
      });

      console.log("Starting to insert " + filename);
      db.collection("caitlintestcollection").insertMany(features, (err, res) => {
        if (err) throw err;
        var totalTime = process.hrtime()[0] - startTime;
        console.log("All entries from " + filename + " inserted ok. Elapsed time is " + totalTime + " seconds.");
      });
    });
  })
});
{'fraction_non_alphanumeric': 0.14197148171109733, 'fraction_numerical': 0.038437693738375696, 'mean_word_length': 4.3250825082508255, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5741432', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1219, 'n_words': 173}