<commit_before>function loadUnicodeData(completion) {
	DataZip = new JSZip();
	JSZipUtils.getBinaryContent('data.zip', function(err, data) {
		if (err) {
			throw err;
		}
		DataZip.loadAsync(data).then(function() {
			completion();
		});
	});
}

function requestAsync(url, before, each, after) {
	console.log(url);
	DataZip.file(url).async('string').then(function(str) {
		var lines = str.split('\n');
		if (before)
			before(lines);
		if (each) {
			for (var i = 0; i < lines.length; ++i) {
				var line = lines[i];
				if (line.length == 0 || line[0] == '#')
					continue;
				if (line.indexOf('#') != -1) {
					line = line.substring(0, line.indexOf('#'));
				}
				each(line);
			}
		}
		if (after) {
			after();
		}
	});
}

function deleteUnicodeData() {
	delete DataZip;
}

function callMultipleAsync(functions, completion) {
	var count = 0;
	var callback = function() {
		++count;
		if (count == functions.length) {
			completion();
		}
	}
	for (var i = 0; i < functions.length; ++i) {
		functions[i](callback);
	}
}<commit_msg>Remove debug logging<commit_after>function loadUnicodeData(completion) {
	DataZip = new JSZip();
	JSZipUtils.getBinaryContent('data.zip', function(err, data) {
		if (err) {
			throw err;
		}
		DataZip.loadAsync(data).then(function() {
			completion();
		});
	});
}

function requestAsync(url, before, each, after) {
	DataZip.file(url).async('string').then(function(str) {
		var lines = str.split('\n');
		if (before)
			before(lines);
		if (each) {
			for (var i = 0; i < lines.length; ++i) {
				var line = lines[i];
				if (line.length == 0 || line[0] == '#')
					continue;
				if (line.indexOf('#') != -1) {
					line = line.substring(0, line.indexOf('#'));
				}
				each(line);
			}
		}
		if (after) {
			after();
		}
	});
}

function deleteUnicodeData() {
	delete DataZip;
}

function callMultipleAsync(functions, completion) {
	var count = 0;
	var callback = function() {
		++count;
		if (count == functions.length) {
			completion();
		}
	}
	for (var i = 0; i < functions.length; ++i) {
		functions[i](callback);
	}
}
{'fraction_non_alphanumeric': 0.18992248062015504, 'fraction_numerical': 0.006782945736434108, 'mean_word_length': 3.3565400843881856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1109505', 'n_tokens_mistral': 871, 'n_tokens_neox': 747, 'n_words': 161}