<commit_before>var logger = require('tracer').console();
var MongoClient = require('mongodb').MongoClient;

function DataStore(config){
	var ds =this;

	ds.config = config;

	ds.id_index = 0;
	ds.id_last_index = 0;
	ds.getting_id_index = false;
	ds.id_callbacks = [];

	ds.connected = false;
	ds.connecting = false;
	ds.connect_callbacks = [];
}

DataStore.id_group_size = 100;

DataStore.prototype.call_connect_callbacks = function(){
	var ds = this;

	logger.log("CALLING CONNECT CALLBACKS");
	for(var i = 0; i < ds.connect_callbacks.length; i++){
		ds.connect_callbacks[i]();
	}
}

DataStore.prototype.connect = function(callback) {
	var ds = this;

	if(ds.connected){
		callback();
		return;
	}
	ds.connect_callbacks.push(callback);
	if(ds.connecting){
		return;
	}
	ds.connecting = true;

	MongoClient.connect(ds.config.address, function(err, mongo_connection) {
	    if (err){
	    	logger.log(err);
	    	throw err;
	    }

	    ds.mongo_connection = mongo_connection;

	    ds.object_collection = ds.mongo_connection.collection('objects');
	    ds.config_collection = ds.mongo_connection.collection('config');
	    ds.type_collection = ds.mongo_connection.collection('types');
	    ds.users_collection = ds.mongo_connection.collection('users');

	    ds.object_collection.ensureIndex( { full_path: 1 }, { unique: true }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex( { path: 1 }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex( { name: 1 }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex(
	    	{ 
	    		"$**": "text"
	    	},
	    	{
	    		name: "AllTextIndex"
	    	},
	    	function(){
	    		logger.log(arguments);
			    ds.connected = true;
			    ds.call_connect_callbacks();
			}
		);

	    // mongo_connection.runCommand({shardCollection:"minods.objects", key: { full_path: 1 }}, function(err,res){
	    // 	logger.log(err);
	    // 	logger.log(res);
	    // })
	});
};

DataStore.prototype.close = function(callback){
	var ds = this;

	if(ds.mongo_connection){
		ds.mongo_connection.close(callback);
	} else {
		if(callback){
			callback();
		}
	}
}

DataStore.prototype.get_id = function(callback){
	var ds = this;

	if(ds.id_index < ds.id_last_index){
		var this_id = ds.id_index++;
		callback(this_id);
		return;
	}

	//Need new ids
	
	//Add the callback to the queue
	ds.id_callbacks.push(callback);
	
	if(ds.getting_id_index){
		//Already retrieving - add to callback queue
		return;
	}

	//This function will start the request
	ds.getting_id_index = true;

	ds.config_collection.insert({"_id":"id_index", "index":DataStore.id_group_size},function(err,res){
		if(err){
			// The index already exists

			ds.config_collection.findAndModify(
				{_id:"id_index"},
				[],
				{$inc:{"index":DataStore.id_group_size}},
				{},
			function(error,response){
	    		ds.id_last_index = response.value.index;<commit_msg>Fixed id incremention in mongodb<commit_after>var logger = require('tracer').console();
var MongoClient = require('mongodb').MongoClient;

function DataStore(config){
	var ds =this;

	ds.config = config;

	ds.id_index = 0;
	ds.id_last_index = 0;
	ds.getting_id_index = false;
	ds.id_callbacks = [];

	ds.connected = false;
	ds.connecting = false;
	ds.connect_callbacks = [];
}

DataStore.id_group_size = 100;

DataStore.prototype.call_connect_callbacks = function(){
	var ds = this;

	logger.log("CALLING CONNECT CALLBACKS");
	for(var i = 0; i < ds.connect_callbacks.length; i++){
		ds.connect_callbacks[i]();
	}
}

DataStore.prototype.connect = function(callback) {
	var ds = this;

	if(ds.connected){
		callback();
		return;
	}
	ds.connect_callbacks.push(callback);
	if(ds.connecting){
		return;
	}
	ds.connecting = true;

	MongoClient.connect(ds.config.address, function(err, mongo_connection) {
	    if (err){
	    	logger.log(err);
	    	throw err;
	    }

	    ds.mongo_connection = mongo_connection;

	    ds.object_collection = ds.mongo_connection.collection('objects');
	    ds.config_collection = ds.mongo_connection.collection('config');
	    ds.type_collection = ds.mongo_connection.collection('types');
	    ds.users_collection = ds.mongo_connection.collection('users');

	    ds.object_collection.ensureIndex( { full_path: 1 }, { unique: true }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex( { path: 1 }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex( { name: 1 }, function(err,res){
	    	logger.log(err, res);
	    })

	    ds.object_collection.ensureIndex(
	    	{ 
	    		"$**": "text"
	    	},
	    	{
	    		name: "AllTextIndex"
	    	},
	    	function(){
	    		logger.log(arguments);
			    ds.connected = true;
			    ds.call_connect_callbacks();
			}
		);

	    // mongo_connection.runCommand({shardCollection:"minods.objects", key: { full_path: 1 }}, function(err,res){
	    // 	logger.log(err);
	    // 	logger.log(res);
	    // })
	});
};

DataStore.prototype.close = function(callback){
	var ds = this;

	if(ds.mongo_connection){
		ds.mongo_connection.close(callback);
	} else {
		if(callback){
			callback();
		}
	}
}

DataStore.prototype.get_id = function(callback){
	var ds = this;

	if(ds.id_index < ds.id_last_index){
		var this_id = ds.id_index++;
		callback(this_id);
		return;
	}

	//Need new ids
	
	//Add the callback to the queue
	ds.id_callbacks.push(callback);
	
	if(ds.getting_id_index){
		//Already retrieving - add to callback queue
		return;
	}

	//This function will start the request
	ds.getting_id_index = true;

	ds.config_collection.insert({"_id":"id_index", "index":DataStore.id_group_size},function(err,res){
		if(err){
			// The index already exists

			ds.config_collection.findAndModify(
				{_id:"id_index"},
				[],
				{"$inc":{"index":DataStore.id_group_size}},
				{"new":true},
			function(error,response){
	    		ds.id_last_index = response.value.index;
{'fraction_non_alphanumeric': 0.15039225504924053, 'fraction_numerical': 0.0033383408446002335, 'mean_word_length': 3.4550185873605948, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '83747', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2085, 'n_words': 344}