<commit_before>

var DB = function () {
	var me = this;
	var dataList = [];
	var geoList = [];
	
	me.addData = function (data, options) {
		extend(options, {
			filterFunction: function () { return true },
			geoName: '?'
		});
		
		var result = [];
		data.forEach(function (obj) {
			if (options.filterFunction(obj)) {
				var value = options.valueFunction(obj);
				var geoId = options.geoIdFunction(obj);
				if ((value || (value === 0)) && (geoId !== undefined)) {
					result.push({
						value: value,
						geoId: geoId
					});
				}
			}
		});
		
		if (result.length > 0) {
			dataList.push({
				data:result,
				options:options
			});
		} else {
			console.log(options.tableName + ' (' + options.year + ') wurde nicht hinzugefügt, weil leer');
		}
	}
	
	me.addGeo = function (data, options) {
		extend(options, {
			synonyms:[]
		});
		
		var index = {};
		for (var i = 0; i < data.length; i++) index['_'+options.geoIdFunction(data[i].properties)] = i;
		
		for (var i = 0; i < options.synonyms.length; i++) {
			var synonymList = options.synonyms[i];
			var synonymId = index['_'+synonymList[0]];
			if (synonymId === undefined) console.error('Unbekannt: "' + synonymList[0] + '"');
			for (var j = 1; j < synonymList.length; j++) {
				index['_'+synonymList[j]] = synonymId;
			}
		}
		
		geoList.push({data:data, options:options, index:index});
	}
	
	me.match = function () {
		for (var i = 0; i < dataList.length; i++) {
			var data = dataList[i].data;
			var geo = getGeoByName(dataList[i].options.geoName);
			var index = geo.index;
			for (var j = 0; j < data.length; j++) {
				var geoIndex = index['_'+data[j].geoId];
				if (geoIndex === undefined) console.log('Was ist "'+data[j].geoId+'"?');
				data[j].geoIndex = geoIndex;
			}
		}
	}
	
	me.exportJSON = function () {
		return {
			data:dataList,
			geo:geoList
		}
	}
	
	function getGeoByName(geoName) {
		for (var i = 0; i < geoList.length; i++) {
			if (geoList[i].options.geoName == geoName) return geoList[i];
		}
		return undefined;
	}
	
	return me;
}

exports.DB = DB;
exports.extend = extend;
exports.merge = merge;

function extend(obj1, obj2) {
	Object.keys(obj2).forEach(function(key) {
		if (obj1[key] === undefined) obj1[key] = obj2[key];
	});
}

function merge(obj1, obj2) {
	var r = {};
	Object.keys(obj2).forEach(function(key) { r[key] = obj2[key] });
	Object.keys(obj1).forEach(function(key) { r[key] = obj1[key] });
	return r;
}<commit_msg>fixed min-max-problem in node.js<commit_after>

var DB = function () {
	var me = this;
	var dataList = [];
	var geoList = [];
	
	me.addData = function (data, options) {
		extend(options, {
			filterFunction: function () { return true },
			geoName: '?'
		});
		
		var result = [];
		var minValue =  1e10;
		var maxValue = -1e10;
		
		data.forEach(function (obj) {
			if (options.filterFunction(obj)) {
				var value = options.valueFunction(obj);
				var geoId = options.geoIdFunction(obj);
				if ((value || (value === 0)) && (geoId !== undefined)) {
					result.push({
						value: value,
						geoId: geoId
					});
					if (minValue > value) minValue = value;
					if (maxValue < value) maxValue = value;
				}
			}
		});
		
		if (options.minValue === undefined) options.minValue = minValue;
		if (options.maxValue === undefined) options.maxValue = maxValue;
		
		if (result.length > 0) {
			dataList.push({
				data:result,
				options:options
			});
		} else {
			console.log(options.tableName + ' (' + options.year + ') wurde nicht hinzugefügt, weil leer');
		}
	}
	
	me.addGeo = function (data, options) {
		extend(options, {
			synonyms:[]
		});
		
		var index = {};
		for (var i = 0; i < data.length; i++) index['_'+options.geoIdFunction(data[i].properties)] = i;
		
		for (var i = 0; i < options.synonyms.length; i++) {
			var synonymList = options.synonyms[i];
			var synonymId = index['_'+synonymList[0]];
			if (synonymId === undefined) console.error('Unbekannt: "' + synonymList[0] + '"');
			for (var j = 1; j < synonymList.length; j++) {
				index['_'+synonymList[j]] = synonymId;
			}
		}
		
		geoList.push({data:data, options:options, index:index});
	}
	
	me.match = function () {
		for (var i = 0; i < dataList.length; i++) {
			var data = dataList[i].data;
			var geo = getGeoByName(dataList[i].options.geoName);
			var index = geo.index;
			for (var j = 0; j < data.length; j++) {
				var geoIndex = index['_'+data[j].geoId];
				if (geoIndex === undefined) console.log('Was ist "'+data[j].geoId+'"?');
				data[j].geoIndex = geoIndex;
			}
		}
	}
	
	me.exportJSON = function () {
		return {
			data:dataList,
			geo:geoList
		}
	}
	
	function getGeoByName(geoName) {
		for (var i = 0; i < geoList.length; i++) {
			if (geoList[i].options.geoName == geoName) return geoList[i];
		}
		return undefined;
	}
	
	return me;
}

exports.DB = DB;
exports.extend = extend;
exports.merge = merge;

function extend(obj1, obj2) {
	Object.keys(obj2).forEach(function(key) {
		if (obj1[key] === undefined) obj1[key] = obj2[key];
	});
}

function merge(obj1, obj2) {
	var r = {};
	Object.keys(obj2).forEach(function(key) { r[key] = obj2[key] });
	Object.keys(obj1).forEach(function(key) { r[key] = obj1[key] });
	return r;
}
{'fraction_non_alphanumeric': 0.1812899477048228, 'fraction_numerical': 0.00968429207824908, 'mean_word_length': 3.1949634443541837, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4763831', 'n_tokens_mistral': 2229, 'n_tokens_neox': 1894, 'n_words': 413}