<commit_before>// Load in MaxMind's IP address block to country code database.
// Either the free or paid one will work.
//
// Provide the location of the country code CSV as the main command line argument.
//
// Run this script from the root dir of the project.

var setupCollection = function(db) {
  // 1) connect to the collection, 
  // 2) clear it,
  // 3) ensure indexes
  // 4) pass it on!

  db.collection("blocks", function(error, collection) {
    collection.remove({}, {safe: true}, function(err, result) {
      console.log("Cleared " + result + " IP block rows.");

      collection.ensureIndex({ip_start: 1}, {}, function(err, indexName) {
        console.log("Created/ensured index " + indexName + " on blocks");
        readCSV(collection);
      });
    });
  });
};

// read in the given CSV, parse each row, store it
var readCSV = function(collection) {
  var filename = args._[0];
  console.log("Reading CSV from " + filename + ":");
  csv()
    .from.stream(fs.createReadStream(filename))
    .on('record', function(row, index) {
      doc = {
        ip_start: new mongo.mongodb.Long(row[2]),
        ip_end: new mongo.mongodb.Long(row[3]),
        country: row[4]
      };

      console.log("#" + index + " " + JSON.stringify(doc));

      collection.insert(doc, {}, function(err, result) {
        if (err) console.log("Error inserting row: " + err);<commit_msg>Complain without a CSV file<commit_after>// Load in MaxMind's IP address block to country code database.
// Either the free or paid one will work.
//
// Provide the location of the country code CSV as the main command line argument.
//
// Run this script from the root dir of the project.

var setupCollection = function(db) {
  // 1) connect to the collection, 
  // 2) clear it,
  // 3) ensure indexes
  // 4) pass it on!

  db.collection("blocks", function(error, collection) {
    collection.remove({}, {safe: true}, function(err, result) {
      console.log("Cleared " + result + " IP block rows.");

      collection.ensureIndex({ip_start: 1}, {}, function(err, indexName) {
        console.log("Created/ensured index " + indexName + " on blocks");
        readCSV(collection);
      });
    });
  });
};

// read in the given CSV, parse each row, store it
var readCSV = function(collection) {
  var filename = args._[0];

  if (!filename) {
    console.log("Supply a CSV filename as a command lne argument.")
    process.exit(1);
  }


  console.log("Reading CSV from " + filename + ":");
  csv()
    .from.stream(fs.createReadStream(filename))
    .on('record', function(row, index) {
      doc = {
        ip_start: new mongo.mongodb.Long(row[2]),
        ip_end: new mongo.mongodb.Long(row[3]),
        country: row[4]
      };

      console.log("#" + index + " " + JSON.stringify(doc));

      collection.insert(doc, {}, function(err, result) {
        if (err) console.log("Error inserting row: " + err);
{'fraction_non_alphanumeric': 0.14974182444061962, 'fraction_numerical': 0.006540447504302926, 'mean_word_length': 2.890227576974565, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4285823', 'n_tokens_mistral': 923, 'n_tokens_neox': 873, 'n_words': 275}