<commit_before>var fs = require('fs');

// db.states.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.entity_url.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.cities.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.entities.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )

// Retrieve
var database;

exports.initialize_db = function(callback) {
      var MongoClient = require('mongodb').MongoClient;
            
      MongoClient.connect("mongodb://localhost:27017/cnes", function (err, db) {
            if (err) {
                  return console.dir(err);
            }

            database = db

            callback();
      });    
}

function save_to_db(json, collection_name) {
      if(json) 
      {
            // Connect to the db
            var collection = database.collection(collection_name);
            collection.insert(json, function (err, inserted) {
                  if (err) {
                        console.log(err);
                  } else {
                        console.log("salvo com sucesso!")
                        json = null;
                  }

                  collection = null;
                  global.gc();
            });
      }
}

exports.save_entity = function (entity) {
      var collection_name = 'entities'
      save_to_db(entity, collection_name);
}

exports.save_states = function (states) {
      var collection_name = 'states'
      save_to_db(states, collection_name);
}

exports.save_cities = function (cities) {
      var collection_name = 'cities'
      save_to_db(cities, collection_name);
}

exports.save_entity_url = function (url) {
      var collection_name = 'entity_url';
      save_to_db(url, collection_name)
}

exports.delete_from_db = function (json, collection_name) {
      var collection = database.collection(collection_name);

      collection.delete(json, function (error, inserted) {
            if (error) {
                  console.log(error);
            }
      });
}

exports.queryByObject = function (json, collection_name, callback) {
      var collection = database.collection(collection_name);
      collection.findOne(json, function(err, result_json){
            callback(result_json);

            result_json = null
            collection = null
            global.gc();
      });
}

exports.getAllCities = function (callback) {
      var collection = database.collection('cities');
      collection.find().sort({"$natural": -1}, function(err, result_cursor){
            callback(result_cursor);
            
            result_cursor = null
            collection = null
            global.gc();
      });
}

exports.isEntityAlreadyDownloaded = function(json, callback)
{
      var callbackConsult = function(documentFromDb)
      {
            var isAlreadyDownloaded = false;

            if(documentFromDb) {
                  isAlreadyDownloaded = true;            
            }

            callback(documentFromDb, isAlreadyDownloaded);
      }

      if(json) {
            if(json.url) {
                  var jsonToConsult = {
                        'url' : json.url
                  }

                  this.queryByObject(jsonToConsult, 'entities', callbackConsult);
            }            
      }
}

exports.get_all_entities = function(callback) {
      var collection = database.collection("entity_url");
      var all_entities = collection.find();

      callback(all_entities);
}
<commit_msg>fixes query to get only not downloaded registers<commit_after>var fs = require('fs');

// db.states.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.entity_url.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.cities.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )
// db.entities.ensureIndex( { "url": 1 }, { unique: true , dropDups: true} )

// Retrieve
var database;

exports.initialize_db = function(callback) {
      var MongoClient = require('mongodb').MongoClient;
            
      MongoClient.connect("mongodb://localhost:27017/cnes", function (err, db) {
            if (err) {
                  return console.dir(err);
            }

            database = db

            callback();
      });    
}

function save_to_db(json, collection_name) {
      if(json) 
      {
            // Connect to the db
            var collection = database.collection(collection_name);
            collection.insert(json, function (err, inserted) {
                  if (err) {
                        console.log(err);
                  } else {
                        console.log("salvo com sucesso!")
                        json = null;
                  }

                  collection = null;
                  global.gc();
            });
      }
}

exports.save_entity = function (entity) {
      var collection_name = 'entities'
      save_to_db(entity, collection_name);
}

exports.save_states = function (states) {
      var collection_name = 'states'
      save_to_db(states, collection_name);
}

exports.save_cities = function (cities) {
      var collection_name = 'cities'
      save_to_db(cities, collection_name);
}

exports.save_entity_url = function (url) {
      var collection_name = 'entity_url';
      save_to_db(url, collection_name)
}

exports.delete_from_db = function (json, collection_name) {
      var collection = database.collection(collection_name);

      collection.delete(json, function (error, inserted) {
            if (error) {
                  console.log(error);
            }
      });
}

exports.queryByObject = function (json, collection_name, callback) {
      var collection = database.collection(collection_name);
      collection.findOne(json, function(err, result_json){
            callback(result_json);

            result_json = null
            collection = null
            global.gc();
      });
}

exports.getAllCities = function (callback) {
      var collection = database.collection('cities');
      collection.find().sort({"$natural": -1}, function(err, result_cursor){
            callback(result_cursor);
            
            result_cursor = null
            collection = null
            global.gc();
      });
}

exports.isEntityAlreadyDownloaded = function(json, callback)
{
      var callbackConsult = function(documentFromDb)
      {
            var isAlreadyDownloaded = false;

            if(documentFromDb) {
                  isAlreadyDownloaded = true;            
            }

            callback(documentFromDb, isAlreadyDownloaded);
      }

      if(json) {
            if(json.url) {
                  var jsonToConsult = {
                        'url' : json.url
                  }

                  this.queryByObject(jsonToConsult, 'entities', callbackConsult);
            }            
      }
}

exports.get_all_entities = function(callback) {
      var collection = database.collection("entities");
      
      collection.find({}, { "url" : 1, "_id" : 0 }, function(error, result_cursor){
            result_cursor.toArray(function(err, urls_objects_arrays) {
                  var all_entities_urls = []

                  for (var i = 0; i < urls_objects_arrays.length; i++) {
                        all_entities_urls.push(urls_objects_arrays[i].url)
                  };

                  var url_collection = database.collection("entity_url");
                  var all_entities = url_collection.find({ "url": { $nin: all_entities_urls } });
                  
                  all_entities.count(function(e, count){
                        callback(all_entities);
                  })
            });
      });
}

{'fraction_non_alphanumeric': 0.11362152278960429, 'fraction_numerical': 0.003003787384092987, 'mean_word_length': 1.8500186081131373, 'pattern_counts': {'":': 11, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7668345', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2077, 'n_words': 410}