<commit_before>Log.setLogLevel(Log.debug);
var TIMEOUT_MS = 10000;

function getFileRange(url, start, end, callback) {
	var xhr = new XMLHttpRequest();
	var range= '';
	xhr.open("GET", url, true);
	xhr.responseType = "arraybuffer";
	if (start !== 0 || end !== Infinity) {
		range = 'bytes=' + start + '-' + (end == Infinity ? '':end);
		xhr.setRequestHeader('Range', range);
	}
	Log.info("XHR", "Getting resource at "+url+" range: "+range);
	xhr.onreadystatechange = function (e) { 
		var buffer;
		if ((xhr.status == 200 || xhr.status == 206 || xhr.status == 304 || xhr.status == 416) && xhr.readyState == this.DONE) {
			buffer = xhr.response;
			buffer.fileStart = start;
			if (!buffer.fileStart) {
				// IE does not support adding properties to an ArrayBuffer generated by XHR
				buffer = buffer.slice(0);
				buffer.fileStart = start;
			}
			callback(buffer);
		}
	};
	xhr.send();
}

function getFile(url, callback) {
	getFileRange(url, 0, Infinity, callback);
}

function checkBoxData(assert, box, data) {
	var i;
	assert.ok(box, "Found "+data.type+" box");
	for (var prop in data) {
		if ([ "sizePosition", "start", "fileStart"].indexOf(prop) > -1) {
			continue;<commit_msg>improve logging in qunit<commit_after>Log.setLogLevel(Log.debug);
var TIMEOUT_MS = 10000;

function getFileRange(url, start, end, callback) {
	var xhr = new XMLHttpRequest();
	var range;
	xhr.open("GET", url, true);
	xhr.responseType = "arraybuffer";
	if (start !== 0 || end !== Infinity) {
		range = 'bytes=' + start + '-' + (end == Infinity ? '':end);
		xhr.setRequestHeader('Range', range);
	}
	Log.info("XHR", "Getting resource at "+url+(range ? " range: "+range : ""));
	xhr.onreadystatechange = function (e) { 
		var buffer;
		if ((xhr.status == 200 || xhr.status == 206 || xhr.status == 304 || xhr.status == 416) && xhr.readyState == this.DONE) {
			buffer = xhr.response;
			buffer.fileStart = start;
			if (!buffer.fileStart) {
				// IE does not support adding properties to an ArrayBuffer generated by XHR
				buffer = buffer.slice(0);
				buffer.fileStart = start;
			}
			callback(buffer);
		}
	};
	xhr.send();
}

function getFile(url, callback) {
	getFileRange(url, 0, Infinity, callback);
}

function checkBoxData(assert, box, data) {
	var i;
	assert.ok(box, "Found "+data.type+" box");
	for (var prop in data) {
		if ([ "sizePosition", "start", "fileStart"].indexOf(prop) > -1) {
			continue;
{'fraction_non_alphanumeric': 0.1758563074352548, 'fraction_numerical': 0.017543859649122806, 'mean_word_length': 3.9381443298969074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7018118', 'n_tokens_mistral': 939, 'n_tokens_neox': 835, 'n_words': 218}