<commit_before>var MongoClient = require('mongodb').MongoClient;
exports.generate = function (converter, url, n, collection, callback) {
	console.log('--GENERATE USER--');
	generateDB(converter, url, n, collection, callback);
};

function generateDB(converter, url, n, collection, callback) {
	MongoClient.connect(url)
			.then(function (db) {
				console.log('[MongoDB] connected');
				var count = 0;
				converter.toIDs(['_isUser', 'name', 'age', 'iq', 'gender', 'height', '_segments'], function (ids) {
					for (var i = 0; i < n; i++) {
						count++;
						var user = generateUsers(ids);
						db.collection(collection).insertOne(user)
								.then(function (results) {
									count--;
									if (count % 1000 == 0)
										console.log((n - count) + ' records');

									if (count == 0) {
										db.close();
										console.log('Done');
										callback();
									}
								}).catch(function (err) {
							console.log('[MongoDB] insert err', err.message);
						});
					}
				});

				// Listen for some events
				db.on('reconnect', function (data) {
					console.log(data);
					console.log('[MongoDB] reconnect success');
				});
				db.on('error', function (err) {
					console.log('[MongoDB] error', err.message);
				});
				db.on('close', function (err) {
					console.log('[MongoDB] disconnected');
				});
			}).catch(function (err) {
		console.error("[MongoDB]", err.message);
		setTimeout(function () {
			generateDB(url)<commit_msg>add _mtid to user<commit_after>var MongoClient = require('mongodb').MongoClient;
exports.generate = function (converter, url, n, collection, callback) {
	console.log('--GENERATE USER--');
	generateDB(converter, url, n, collection, callback);
};

function generateDB(converter, url, n, collection, callback) {
	MongoClient.connect(url)
			.then(function (db) {
				console.log('[MongoDB] connected');
				var count = 0;
				converter.toIDs(['_isUser', 'name', 'age', 'iq', 'gender', 'height', '_segments'], function (ids) {
					for (var i = 0; i < n; i++) {
						count++;
						var user = generateUsers(ids);
						db.collection(collection).insertOne(user)
								.then(function (results) {
									var newuser = results.ops[0];
									newuser._mtid = newuser._id;
									db.collection(collection).updateOne({_id: newuser._id}, newuser, function () {
										count--;
										if (count % 1000 == 0)
											console.log((n - count) + ' records');

										if (count == 0) {
											db.close();
											console.log('Done');
											callback();
										}
									});
								}).catch(function (err) {
							console.log('[MongoDB] insert err', err.message);
						});
					}
				});

				// Listen for some events
				db.on('reconnect', function (data) {
					console.log(data);
					console.log('[MongoDB] reconnect success');
				});
				db.on('error', function (err) {
					console.log('[MongoDB] error', err.message);
				});
				db.on('close', function (err) {
					console.log('[MongoDB] disconnected');
				});
			}).catch(function (err) {
		console.error("[MongoDB]", err.message);
		setTimeout(function () {
			generateDB(url)
{'fraction_non_alphanumeric': 0.17891373801916932, 'fraction_numerical': 0.005431309904153354, 'mean_word_length': 2.7097156398104265, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3799603', 'n_tokens_mistral': 1393, 'n_tokens_neox': 987, 'n_words': 215}