<commit_before>var readline = require("readline");
var mongoose = require("mongoose");
var db = require("./databaseBackendSequelize");
var db2 = require("./database");
var utils = require("./utils");
var async = require("async");


var OLD_MARKER_COLOURS = { blue: "8da8f6", green: "90ee90", gold: "ffd700", red: "ff0000" };

var ObjectId = mongoose.Schema.Types.ObjectId;

var positionType = {
	lon: Number,
	lat: Number
};

var bboxType = {
	top: Number,
	right: Number,
	bottom: Number,
	left: Number
};

var markerSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	position : positionType,
	name : { type: String, default: "Untitled marker" },
	description : String,
	colour : { type: String, default: "ff0000" }
});

var lineSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	points : [positionType],
	mode : { type: String, default: "" },
	colour : { type: String, default: "0000ff" },
	width : { type: Number, default: 4 },
	description : String,
	name : { type: String, default: "Untitled line" },
	distance : Number,
	time : Number
});

var linePointsSchema = mongoose.Schema(utils.extend({ }, positionType, {
	zoom : Number,
	idx : Number,
	_line : { type: ObjectId, ref: "Line" }
}));

var viewSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	name : String,
	baseLayer : String,
	layers : [String],
	view : bboxType
});

var padSchema = mongoose.Schema({
	_id : String,
	defaultView : { type: ObjectId, ref: "View" },
	name: { type: String, default: "New FacilPad" },
	writeId : String
});

var typeSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	name : String,
	type : { type: String, enum: [ "marker", "line" ] },
	fields : [ Object ]
});

var Marker = mongoose.model("Marker", markerSchema);
var Line = mongoose.model("Line", lineSchema);
var LinePoints = mongoose.model("LinePoints", linePointsSchema);
var View = mongoose.model("View", viewSchema);
var Pad = mongoose.model("Pad", padSchema);
var Type = mongoose.model("Type", typeSchema);


function migrateData(title, stream, deal, callback) {
	console.log();
	console.log();
	console.log("Migrating "+title);
	console.log();

	var number = 0;
	var ended = false;

	stream.on("data", function(data) {
		data = JSON.parse(JSON.stringify(data));
		data.id = data._id;
		delete data._id;

		var queries = deal(data);
		var outstandingQueries = queries.length;

		if(queries.length == 0)
			return check();

		number++;

		for(var i=0; i<queries.length; i++) {
			queries[i].complete(function(err) {
				err && console.error(err);

				if(--outstandingQueries == 0) {
					number--;
					check();
				}
			});
		}
	});

	stream.on("error", function(err) {
		console.error(err);
		ended = true;
		check();
	});

	stream.on("end", function() {
		ended = true;
		check();
	});

	function check() {
		if(ended && number == 0)
			callback();
	}
}

var DEFAULT_MARKER_TYPE = db2._defaultTypes[0];
var DEFAULT_LINE_TYPE = db2._defaultTypes[1];


var rl = readline.createInterface({
	input: process.stdin,
	output: process.stdout
});

console.log("Target SQL database will be cleared!");
rl.question("Please enter the MongoDB connection string [mongodb://localhost/facilpad]: ", function(connectionString) {<commit_msg>Pausing and resuming stream to avoid memory overflow in database migration<commit_after>var readline = require("readline");
var mongoose = require("mongoose");
var db = require("./databaseBackendSequelize");
var db2 = require("./database");
var utils = require("./utils");
var async = require("async");


var OLD_MARKER_COLOURS = { blue: "8da8f6", green: "90ee90", gold: "ffd700", red: "ff0000" };

var ObjectId = mongoose.Schema.Types.ObjectId;

var positionType = {
	lon: Number,
	lat: Number
};

var bboxType = {
	top: Number,
	right: Number,
	bottom: Number,
	left: Number
};

var markerSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	position : positionType,
	name : { type: String, default: "Untitled marker" },
	description : String,
	colour : { type: String, default: "ff0000" }
});

var lineSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	points : [positionType],
	mode : { type: String, default: "" },
	colour : { type: String, default: "0000ff" },
	width : { type: Number, default: 4 },
	description : String,
	name : { type: String, default: "Untitled line" },
	distance : Number,
	time : Number
});

var linePointsSchema = mongoose.Schema(utils.extend({ }, positionType, {
	zoom : Number,
	idx : Number,
	_line : { type: ObjectId, ref: "Line" }
}));

var viewSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	name : String,
	baseLayer : String,
	layers : [String],
	view : bboxType
});

var padSchema = mongoose.Schema({
	_id : String,
	defaultView : { type: ObjectId, ref: "View" },
	name: { type: String, default: "New FacilPad" },
	writeId : String
});

var typeSchema = mongoose.Schema({
	_pad : { type: String, ref: "Pad" },
	name : String,
	type : { type: String, enum: [ "marker", "line" ] },
	fields : [ Object ]
});

var Marker = mongoose.model("Marker", markerSchema);
var Line = mongoose.model("Line", lineSchema);
var LinePoints = mongoose.model("LinePoints", linePointsSchema);
var View = mongoose.model("View", viewSchema);
var Pad = mongoose.model("Pad", padSchema);
var Type = mongoose.model("Type", typeSchema);


function migrateData(title, stream, deal, callback) {
	console.log();
	console.log();
	console.log("Migrating "+title);
	console.log();

	var queue = async.queue(function(data, next) {
		data = JSON.parse(JSON.stringify(data));
		data.id = data._id;
		delete data._id;

		var queries = deal(data);

		async.each(queries, function(it, next) {
			it.complete(function(err) {
				err && console.error(err);
				next();
			});
		}, next);
	}, 5);

	var startStop = function() {
		if(queue.length() == 0)
			stream.resume();
		else
			stream.pause();
	};

	stream.on("data", function(data) {
		queue.push(data, startStop);
		startStop();
	});

	stream.on("end", function() {
		if(queue.running() == 0 && queue.length() == 0)
			callback();
		else
			queue.drain = callback;
	});

	stream.on("error", function(err) {
		queue.kill();
		callback(err);
	});
}

var DEFAULT_MARKER_TYPE = db2._defaultTypes[0];
var DEFAULT_LINE_TYPE = db2._defaultTypes[1];


var rl = readline.createInterface({
	input: process.stdin,
	output: process.stdout
});

console.log("Target SQL database will be cleared!");
rl.question("Please enter the MongoDB connection string [mongodb://localhost/facilpad]: ", function(connectionString) {
{'fraction_non_alphanumeric': 0.17007946210268948, 'fraction_numerical': 0.00993276283618582, 'mean_word_length': 4.347222222222222, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7545005', 'n_tokens_mistral': 2356, 'n_tokens_neox': 2246, 'n_words': 576}